@things-factory/dataset 6.0.66 → 6.0.69

package/helps/dataset/data-summary.md

@@ -0,0 +1,160 @@
+# DataSummary
+
+Paragraphs are separated by a blank line.
+
+2nd paragraph. _Italic_, **bold**, and `monospace`. Itemized lists
+look like:
+
+- this one
+- that one
+- the other one
+
+Note that --- not considering the asterisk --- the actual text
+content starts at 4-columns in.
+
+> Block quotes are
+> written like so.
+>
+> They can span multiple paragraphs,
+> if you like.
+
+Use 3 dashes for an em-dash. Use 2 dashes for ranges (ex., "it's all
+in chapters 12--14"). Three dots ... will be converted to an ellipsis.
+Unicode is supported. ☺
+
+## An h2 header
+
+Here's a numbered list:
+
+1. first item
+2. second item
+3. third item
+
+Note again how the actual text starts at 4 columns in (4 characters
+from the left side). Here's a code sample:
+
+# Let me re-iterate ...
+
+for i in 1 .. 10 { do-something(i) }
+
+As you probably guessed, indented 4 spaces. By the way, instead of
+indenting the block, you can use delimited blocks, if you like:
+
+```
+define foobar() {
+print "Welcome to flavor country!";
+}
+```
+
+(which makes copying & pasting easier). You can optionally mark the
+delimited block for Pandoc to syntax highlight it:
+
+```python
+import time
+# Quick, count to ten!
+for i in range(10):
+# (but not *too* quick)
+time.sleep(0.5)
+print(i)
+```
+
+### An h3 header
+
+Now a nested list:
+
+1. First, get these ingredients:
+
+- carrots
+- celery
+- lentils
+
+2. Boil some water.
+
+3. Dump everything in the pot and follow
+   this algorithm:
+
+find wooden spoon
+uncover pot
+stir
+cover pot
+balance wooden spoon precariously on pot handle
+wait 10 minutes
+goto first step (or shut off burner when done)
+
+Do not bump wooden spoon or it will fall.
+
+Notice again how text always lines up on 4-space indents (including
+that last line which continues item 3 above).
+
+Here's a link to [a website](http://foo.bar), to a [local
+doc](local-doc.html), and to a [section heading in the current
+doc](#an-h2-header). Here's a footnote [^1].
+
+[^1]: Some footnote text.
+
+Tables can look like this:
+
+Name Size Material Color
+
+---
+
+All Business 9 leather brown
+Roundabout 10 hemp canvas natural
+Cinderella 11 glass transparent
+
+Table: Shoes sizes, materials, and colors.
+
+(The above is the caption for the table.) Pandoc also supports
+multi-line tables:
+
+---
+
+Keyword Text
+
+---
+
+red Sunsets, apples, and
+other red or reddish
+things.
+
+green Leaves, grass, frogs
+and other things it's
+not easy being.
+
+---
+
+A horizontal rule follows.
+
+---
+
+Here's a definition list:
+
+apples
+: Good for making applesauce.
+
+oranges
+: Citrus!
+
+tomatoes
+: There's no "e" in tomatoe.
+
+Again, text is indented 4 spaces. (Put a blank line between each
+term and its definition to spread things out more.)
+
+Here's a "line block" (note how whitespace is honored):
+
+| Line one
+| Line too
+| Line tree
+
+and images can be specified like so:
+
+![example image](example-image.jpg 'An exemplary image')
+
+Inline math equation: $\omega = d\phi / dt$. Display
+math should get its own line like so:
+
+$$I = \int \rho R^{2} dV$$
+
+And note that you can backslash-escape any punctuation characters
+which you wish to be displayed literally, ex.: \`foo\`, \*bar\*, etc.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/dataset",
-  "version": "6.0.66",
+  "version": "6.0.69",
   "main": "dist-server/index.js",
   "browser": "dist-client/index.js",
   "things-factory": true,
@@ -35,17 +35,17 @@
     "@operato/shell": "^1.0.1",
     "@operato/styles": "^1.0.0",
     "@operato/utils": "^1.0.1",
-    "@things-factory/auth-base": "^6.0.60",
-    "@things-factory/aws-base": "^6.0.60",
-    "@things-factory/board-service": "^6.0.60",
+    "@things-factory/auth-base": "^6.0.67",
+    "@things-factory/aws-base": "^6.0.67",
+    "@things-factory/board-service": "^6.0.69",
     "@things-factory/env": "^6.0.49",
-    "@things-factory/organization": "^6.0.60",
-    "@things-factory/scheduler-client": "^6.0.60",
-    "@things-factory/shell": "^6.0.60",
-    "@things-factory/work-shift": "^6.0.60",
-    "@things-factory/worklist": "^6.0.60",
+    "@things-factory/organization": "^6.0.67",
+    "@things-factory/scheduler-client": "^6.0.67",
+    "@things-factory/shell": "^6.0.67",
+    "@things-factory/work-shift": "^6.0.67",
+    "@things-factory/worklist": "^6.0.69",
     "cron-parser": "^4.3.0",
     "moment-timezone": "^0.5.40"
   },
-  "gitHead": "be8f5c22cde7454f21d5bfabc0eb092ef59c0701"
+  "gitHead": "d0198bba193b275f29fffd8c3963f556db4dda92"
 }

package/server/controllers/create-data-sample.ts

@@ -57,7 +57,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
   const { domain, user, tx } = context.state
 
   const dataSet = await tx.getRepository(DataSet).findOne({
-    where: { id: dataSample.dataSet.id },
+    where: { domain: { id: domain.id }, id: dataSample.dataSet.id },
     relations: ['dataKeySet']
   })
 

package/server/controllers/generate-data-summary.ts

@@ -0,0 +1,124 @@
+import { Sorting, getQueryBuilderFromListParams } from '@things-factory/shell'
+import { logger } from '@things-factory/env'
+
+import { DataSample } from '../service/data-sample/data-sample'
+import { DataSet } from '../service/data-set/data-set'
+
+import { DataSummary } from '../service/data-summary/data-summary'
+import { DataKeyItem } from '../service/data-key-set/data-key-item-type'
+
+const compareKeys = (dataKeyItems: DataKeyItem[], summary: Partial<DataSummary>, sample: DataSample): boolean => {
+  return dataKeyItems.every((item, index) => {
+    const prop = `key0${index + 1}`
+    return sample[prop] === summary[prop]
+  })
+}
+
+const buildKeysFromSample = (dataKeyItems: DataKeyItem[], sample: DataSample): Partial<DataSummary> => {
+  return dataKeyItems.reduce((sum, item, index) => {
+    const prop = `key0${index + 1}`
+    sum[prop] = sample[prop]
+
+    return sum
+  }, {} as Partial<DataSummary>)
+}
+
+const buildKeySortingList = (dataKeyItems: DataKeyItem[]): Sorting[] => {
+  return dataKeyItems.reduce((sum, item, index) => {
+    const name = `key0${index + 1}`
+    sum.push({ name, desc: true })
+    return sum
+  }, [])
+}
+
+export async function generateDataSummary(
+  dataSetId: string,
+  workDate: string,
+  workShift: string = 'NA',
+  context: ResolverContext
+): Promise<boolean> {
+  const { domain, user, tx } = context.state
+
+  try {
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: { domain: { id: domain.id }, id: dataSetId },
+      relations: ['dataKeySet']
+    })
+
+    const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
+
+    const limit = 100
+    var page = 1
+
+    var summaries: Partial<DataSummary>[] = []
+    var summary: Partial<DataSummary>
+
+    do {
+      const samples = await getQueryBuilderFromListParams({
+        repository: tx.getRepository(DataSample),
+        params: {
+          filters: [
+            { name: 'dataSetId', operator: 'eq', value: dataSetId },
+            { name: 'workDate', operator: 'eq', value: workDate },
+            { name: 'workShift', operator: 'eq', value: workShift }
+          ],
+          pagination: { page, limit },
+          sortings: [...buildKeySortingList(dataKeyItems), { name: 'collectedAt', desc: true }]
+        },
+        domain
+      }).getMany()
+
+      for (const sample of samples) {
+        if (!summary || !compareKeys(dataKeyItems, summary, sample)) {
+          if (summary) {
+            summaries.push(summary)
+          }
+
+          summary = {
+            domain,
+            name: dataSet.name,
+            description: dataSet.description,
+            workDate,
+            workShift,
+            dataSet,
+            ...buildKeysFromSample(dataKeyItems, sample),
+            count: 0,
+            countOoc: 0,
+            countOos: 0,
+            updater: user,
+            creator: user
+          }
+        }
+
+        summary.count++
+        sample.ooc && summary.countOoc++
+        sample.oos && summary.countOos++
+      }
+
+      if (samples.length < limit) {
+        summary && summaries.push(summary)
+        break
+      }
+
+      page++
+    } while (true)
+
+    tx.getRepository(DataSummary).upsert(summaries, [
+      'domain',
+      'dataSet',
+      'key01',
+      'key02',
+      'key03',
+      'key04',
+      'key05',
+      'workDate',
+      'workShift'
+    ])
+
+    return true
+  } catch (e) {
+    logger.error(e)
+  }
+
+  return false
+}

package/server/service/data-summary/data-summary-mutation.ts

@@ -0,0 +1,19 @@
+import { Resolver, Mutation, Arg, Ctx, Directive } from 'type-graphql'
+
+import { DataSummary } from './data-summary'
+
+import { generateDataSummary } from '../../controllers/generate-data-summary'
+
+@Resolver(DataSummary)
+export class DataSummaryMutation {
+  @Directive('@transaction')
+  @Mutation(returns => Boolean, { description: 'To generate new Data Summaries' })
+  async generateDataSummaries(
+    @Arg('dataSetId') dataSetId: string,
+    @Arg('workDate') workDate: string,
+    @Arg('workShift') workShift: string,
+    @Ctx() context: ResolverContext
+  ): Promise<boolean> {
+    return await generateDataSummary(dataSetId, workDate, workShift, context)
+  }
+}

package/server/service/data-summary/data-summary-query.ts

@@ -0,0 +1,96 @@
+import { Resolver, Query, FieldResolver, Root, Args, Arg, Ctx, Directive } from 'type-graphql'
+import { Domain, getQueryBuilderFromListParams, getRepository, ListParam } from '@things-factory/shell'
+import { User } from '@things-factory/auth-base'
+import { DataKeySet } from '../data-key-set/data-key-set'
+import { DataItem } from '../data-set/data-item-type'
+import { DataSet } from '../data-set/data-set'
+import { DataSetHistory } from '../data-set-history/data-set-history'
+import { DataSummary } from './data-summary'
+import { DataSummaryList } from './data-summary-type'
+
+@Resolver(DataSummary)
+export class DataSummaryQuery {
+  @Query(returns => DataSummary!, { nullable: true, description: 'To fetch a DataSummary' })
+  async dataSummary(@Arg('id') id: string, @Ctx() context: ResolverContext): Promise<DataSummary> {
+    const { domain } = context.state
+
+    return await getRepository(DataSummary).findOne({
+      where: { domain: { id: domain.id }, id }
+    })
+  }
+
+  @Query(returns => DataSummaryList, { description: 'To fetch multiple DataSummaries' })
+  async dataSummaries(@Args() params: ListParam, @Ctx() context: ResolverContext): Promise<DataSummaryList> {
+    const { domain } = context.state
+
+    const queryBuilder = getQueryBuilderFromListParams({
+      domain,
+      params,
+      repository: await getRepository(DataSummary),
+      searchables: ['name', 'description']
+    })
+
+    const [items, total] = await queryBuilder.getManyAndCount()
+
+    return { items, total }
+  }
+
+  @Directive('@privilege(category: "data-summary", privilege: "query", domainOwnerGranted: true)')
+  @Query(returns => DataSummaryList, { description: 'To fetch multiple data summaries by data key set' })
+  async dataSummariesByDataKeySet(
+    @Arg('dataKeySetId') dataKeySetId: string,
+    @Args() params: ListParam,
+    @Ctx() context: ResolverContext
+  ): Promise<DataSummaryList> {
+    const { domain } = context.state
+
+    const dataKeySet = await getRepository(DataKeySet).findOneBy({ id: dataKeySetId })
+    const dataKeyItems = dataKeySet?.dataKeyItems || []
+    const searchables = dataKeyItems.map((item, index) => `key0${index + 1}`)
+
+    const queryBuilder = getQueryBuilderFromListParams({
+      repository: getRepository(DataSummary),
+      params,
+      domain,
+      alias: 'dks',
+      searchables: ['name', 'description'].concat(searchables)
+    }).innerJoin('dks.dataSet', 'ds', 'ds.dataKeySet = :dataKeySetId', {
+      dataKeySetId
+    })
+
+    const [items, total] = await queryBuilder.getManyAndCount()
+
+    return { items, total }
+  }
+
+  @FieldResolver(type => [DataItem])
+  async dataItems(@Root() dataSummary: DataSummary): Promise<DataItem[]> {
+    const dataSetHistory: DataSetHistory = await getRepository(DataSetHistory).findOne({
+      where: {
+        originalId: dataSummary.dataSetId,
+        version: dataSummary.dataSetVersion
+      }
+    })
+    return dataSetHistory?.dataItems || []
+  }
+
+  @FieldResolver(type => DataSet)
+  async dataSet(@Root() dataSummary: DataSummary): Promise<DataSet> {
+    return await getRepository(DataSet).findOneBy({ id: dataSummary.dataSetId })
+  }
+
+  @FieldResolver(type => Domain)
+  async domain(@Root() dataSummary: DataSummary): Promise<Domain> {
+    return await getRepository(Domain).findOneBy({ id: dataSummary.domainId })
+  }
+
+  @FieldResolver(type => User)
+  async updater(@Root() dataSummary: DataSummary): Promise<User> {
+    return await getRepository(User).findOneBy({ id: dataSummary.updaterId })
+  }
+
+  @FieldResolver(type => User)
+  async creator(@Root() dataSummary: DataSummary): Promise<User> {
+    return await getRepository(User).findOneBy({ id: dataSummary.creatorId })
+  }
+}

package/server/service/data-summary/data-summary-type.ts

@@ -0,0 +1,40 @@
+import type { FileUpload } from 'graphql-upload/GraphQLUpload.js'
+import GraphQLUpload from 'graphql-upload/GraphQLUpload.js'
+import { ObjectType, Field, InputType, Int, ID, registerEnumType } from 'type-graphql'
+
+import { ObjectRef, ScalarObject } from '@things-factory/shell'
+
+import { DataSummary, DataSummaryDurationType } from './data-summary'
+
+@InputType()
+export class NewDataSummary {
+  @Field()
+  name: string
+
+  @Field({ nullable: true })
+  description?: string
+}
+
+@InputType()
+export class DataSummaryPatch {
+  @Field(type => ID, { nullable: true })
+  id?: string
+
+  @Field({ nullable: true })
+  name?: string
+
+  @Field({ nullable: true })
+  description?: string
+
+  @Field({ nullable: true })
+  cuFlag?: string
+}
+
+@ObjectType()
+export class DataSummaryList {
+  @Field(type => [DataSummary])
+  items: DataSummary[]
+
+  @Field(type => Int)
+  total: number
+}

package/server/service/data-summary/data-summary.ts

@@ -0,0 +1,165 @@
+import {
+  CreateDateColumn,
+  UpdateDateColumn,
+  Entity,
+  Index,
+  Column,
+  RelationId,
+  ManyToOne,
+  PrimaryGeneratedColumn
+} from 'typeorm'
+import { ObjectType, Field, Int, ID, registerEnumType } from 'type-graphql'
+
+import { Domain, ScalarObject } from '@things-factory/shell'
+import { User } from '@things-factory/auth-base'
+
+import { DataItem } from '../data-set/data-item-type'
+import { DataSet } from '../data-set/data-set'
+
+export enum DataSummaryDurationType {
+  Shift = 'shift',
+  Daily = 'daily',
+  Weekly = 'weekly',
+  Monthly = 'monthly',
+  Quarterly = 'quarterly',
+  Yearly = 'yearly',
+  Custom = 'custom'
+}
+
+registerEnumType(DataSummaryDurationType, {
+  name: 'DataSummaryDurationType',
+  description: 'entry type enumeration for data-summary'
+})
+
+@Entity()
+@Index(
+  'ix_data_summary_0',
+  (dataSummary: DataSummary) => [
+    dataSummary.domain,
+    dataSummary.dataSet,
+    dataSummary.key01,
+    dataSummary.key02,
+    dataSummary.key03,
+    dataSummary.key05,
+    dataSummary.key04,
+    dataSummary.workDate,
+    dataSummary.workShift
+  ],
+  { unique: true }
+)
+@Index(
+  'ix_data_summary_1',
+  (dataSummary: DataSummary) => [dataSummary.domain, dataSummary.dataSet, dataSummary.workDate, dataSummary.workShift],
+  { unique: false }
+)
+@ObjectType({ description: 'Entity for DataSummary' })
+export class DataSummary {
+  @PrimaryGeneratedColumn('uuid')
+  @Field(type => ID)
+  readonly id: string
+
+  @ManyToOne(type => Domain)
+  @Field({ nullable: true })
+  domain?: Domain
+
+  @RelationId((dataSummary: DataSummary) => dataSummary.domain)
+  domainId?: string
+
+  @Column()
+  @Field({ nullable: true })
+  name?: string
+
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  description?: string
+
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  type?: DataSummaryDurationType
+
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  workDate?: string
+
+  @Column({ nullable: true, default: 'NA' })
+  @Field({ nullable: true })
+  workShift?: string = 'NA'
+
+  @ManyToOne(type => DataSet)
+  @Field(type => DataSet, { nullable: false })
+  dataSet?: DataSet
+
+  @RelationId((dataSample: DataSummary) => dataSample.dataSet)
+  dataSetId?: string
+
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  dataSetVersion?: number
+
+  @Field(type => [DataItem], { nullable: true })
+  dataItems?: DataItem[]
+
+  @Column('simple-json', { nullable: true })
+  @Field(type => ScalarObject, { nullable: true })
+  summary?: any
+
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  count?: number
+
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  countOoc?: number
+
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  countOos?: number
+
+  @Column({ nullable: true, default: '' })
+  @Field({ nullable: true })
+  key01?: string = ''
+
+  @Column({ nullable: true, default: '' })
+  @Field({ nullable: true })
+  key02?: string = ''
+
+  @Column({ nullable: true, default: '' })
+  @Field({ nullable: true })
+  key03?: string = ''
+
+  @Column({ nullable: true, default: '' })
+  @Field({ nullable: true })
+  key04?: string = ''
+
+  @Column({ nullable: true, default: '' })
+  @Field({ nullable: true })
+  key05?: string = ''
+
+  @CreateDateColumn()
+  @Field({ nullable: true })
+  createdAt?: Date
+
+  @UpdateDateColumn()
+  @Field({ nullable: true })
+  updatedAt?: Date
+
+  @ManyToOne(type => User, { nullable: true })
+  @Field(type => User, { nullable: true })
+  creator?: User
+
+  @RelationId((dataSummary: DataSummary) => dataSummary.creator)
+  creatorId?: string
+
+  @ManyToOne(type => User, { nullable: true })
+  @Field(type => User, { nullable: true })
+  updater?: User
+
+  @RelationId((dataSummary: DataSummary) => dataSummary.updater)
+  updaterId?: string
+}

package/server/service/data-summary/index.ts

@@ -0,0 +1,7 @@
+import { DataSummary } from './data-summary'
+import { DataSummaryQuery } from './data-summary-query'
+import { DataSummaryMutation } from './data-summary-mutation'
+
+export const entities = [DataSummary]
+export const resolvers = [DataSummaryQuery, DataSummaryMutation]
+export const subscribers = []

package/server/service/index.ts

@@ -8,10 +8,12 @@ import {
   resolvers as DataSetHistoryResolvers,
   subscribers as DataSetHistorySubscribers
 } from './data-set-history'
+import { entities as DataSummaryEntities, resolvers as DataSummaryResolvers } from './data-summary'
 import { entities as DataSpecEntities, resolvers as DataSpecResolvers } from './data-spec'
 import { entities as DataArchiveEntities, resolvers as DataArchiveResolvers } from './data-archive'
 
 /* EXPORT ENTITY TYPES */
+export * from './data-summary/data-summary'
 export * from './data-key-set/data-key-set'
 export * from './data-ooc/data-ooc'
 export * from './data-sensor/data-sensor'
@@ -23,6 +25,7 @@ export * from './data-spec/data-spec'
 
 export const entities = [
   /* ENTITIES */
+  ...DataSummaryEntities,
   ...DataKeySetEntities,
   ...DataOocEntities,
   ...DataSensorEntities,
@@ -41,6 +44,7 @@ export const subscribers = [
 export const schema = {
   resolverClasses: [
     /* RESOLVER CLASSES */
+    ...DataSummaryResolvers,
     ...DataKeySetResolvers,
     ...DataOocResolvers,
     ...DataSensorResolvers,