json-fs-query 0.1.0

package/src/data/queries/insertEntity.test.js

@@ -0,0 +1,156 @@
+import { create } from '../introspect/create'
+import { insertEntity } from './insertEntity'
+import { testHelpers } from 'js-build-tools/testHelpers'
+
+const { readFile } = require('fs/promises')
+
+const databasePath = 'insert-entity-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+const definition = [
+  { name: '_id', type: 'string', optional: false, autoGenerate: true },
+  { name: 'bar', type: 'string', optional: true, autoGenerate: false }
+]
+
+const keys = [
+  { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+]
+
+describe('insertEntity', () => {
+  test('inserts a new entity, generating its primary key', async () => {
+    await create('foo', definition, keys)
+    const dataSet = await insertEntity('foo', [{ bar: 'baz' }], {})
+
+    expect(dataSet.foo).toHaveLength(1)
+    const inserted = dataSet.foo[0]
+    expect(typeof inserted._id).toBe('string')
+    expect(inserted.bar).toBe('baz')
+
+    expect(testHelpers.fileExists(`${databasePath}foo/${inserted._id}.json`)).toBeTruthy()
+    const entryFile = JSON.parse(await readFile(`${databasePath}foo/${inserted._id}.json`, 'utf8'))
+    expect(entryFile).toEqual(inserted)
+
+    const recordFile = JSON.parse(await readFile(`${databasePath}foo.json`, 'utf8'))
+    expect(recordFile.entries).toEqual([`${inserted._id}.json`])
+
+    const indexFile = JSON.parse(await readFile(`${databasePath}__indexes/foo/pk__id.json`, 'utf8'))
+    expect(indexFile).toEqual([{ value: inserted._id, record: [`${inserted._id}.json`] }])
+  })
+
+  test('inserts multiple entities from one call', async () => {
+    await create('foo', definition, keys)
+    const dataSet = await insertEntity('foo', [{ bar: 'one' }, { bar: 'two' }], {})
+
+    expect(dataSet.foo).toHaveLength(2)
+    const recordFile = JSON.parse(await readFile(`${databasePath}foo.json`, 'utf8'))
+    expect(recordFile.entries).toHaveLength(2)
+  })
+
+  test('appends onto an existing dataSet entry instead of replacing it', async () => {
+    await create('foo', definition, keys)
+    const dataSet = await insertEntity('foo', [{ bar: 'one' }], { foo: [{ bar: 'existing' }] })
+    expect(dataSet.foo).toHaveLength(2)
+    expect(dataSet.foo[0]).toEqual({ bar: 'existing' })
+  })
+
+  test('applies a field default when no value is given', async () => {
+    const definitionWithDefault = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: true },
+      { name: 'status', type: 'string', optional: false, useDefault: true, defaultValue: 'pending' }
+    ]
+    await create('foo', definitionWithDefault, keys)
+    const dataSet = await insertEntity('foo', [{}], {})
+    expect(dataSet.foo[0].status).toBe('pending')
+  })
+
+  test('throws when a required field has no value, default, or autoGenerate', async () => {
+    const definitionRequiringBar = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: true },
+      { name: 'bar', type: 'string', optional: false, autoGenerate: false }
+    ]
+    await create('foo', definitionRequiringBar, keys)
+    await expect(insertEntity('foo', [{}], {})).rejects.toThrow('Missing required field "bar" for foo')
+  })
+
+  test('throws on a duplicate primary key value instead of writing the file', async () => {
+    const definitionExplicitId = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: false },
+      { name: 'bar', type: 'string', optional: true }
+    ]
+    await create('foo', definitionExplicitId, keys)
+    await insertEntity('foo', [{ _id: 'dupe', bar: 'first' }], {})
+    await expect(insertEntity('foo', [{ _id: 'dupe', bar: 'second' }], {}))
+      .rejects.toThrow('Duplicate value for primary key "_id" on foo')
+  })
+
+  test('coerces a value to match its field\'s declared type', async () => {
+    const definitionWithInt = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: true },
+      { name: 'reps', type: 'int', optional: false, autoGenerate: false }
+    ]
+    await create('foo', definitionWithInt, keys)
+    const dataSet = await insertEntity('foo', [{ reps: '10' }], {})
+    expect(dataSet.foo[0].reps).toBe(10)
+  })
+
+  test('rejects a value that does not match its field\'s declared type', async () => {
+    const definitionWithInt = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: true },
+      { name: 'reps', type: 'int', optional: false, autoGenerate: false }
+    ]
+    await create('foo', definitionWithInt, keys)
+    await expect(insertEntity('foo', [{ reps: 'ten' }], {}))
+      .rejects.toThrow('Value "ten" is not a valid int')
+  })
+
+  describe('foreign keys', () => {
+    const exerciseDefinition = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: true },
+      { name: 'name', type: 'string', optional: true, autoGenerate: false }
+    ]
+    const exerciseKeys = [
+      { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+    ]
+    const workoutDefinition = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: true },
+      { name: 'exercise_id', type: 'string', optional: true, autoGenerate: false }
+    ]
+    const workoutKeys = [
+      { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' },
+      {
+        type: 'foreign',
+        fields: ['exercise_id'],
+        lookup: 'fk_exercise_id.json',
+        references: [{ fields: ['exercises._id'], lookup: 'exercises.json' }]
+      }
+    ]
+
+    test('accepts a foreign key value that exists in the referenced record', async () => {
+      await create('exercises', exerciseDefinition, exerciseKeys)
+      const exercise = await insertEntity('exercises', [{ name: 'Bench Press' }], {})
+      await create('workouts', workoutDefinition, workoutKeys)
+      const result = await insertEntity('workouts', [{ exercise_id: exercise.exercises[0]._id }], {})
+      expect(result.workouts[0].exercise_id).toBe(exercise.exercises[0]._id)
+    })
+
+    test('rejects a foreign key value that does not exist in the referenced record', async () => {
+      await create('exercises', exerciseDefinition, exerciseKeys)
+      await create('workouts', workoutDefinition, workoutKeys)
+      await expect(insertEntity('workouts', [{ exercise_id: 'does-not-exist' }], {}))
+        .rejects.toThrow('Foreign key "exercise_id" on workouts references a nonexistent exercises._id = does-not-exist')
+    })
+
+    test('skips the check when an optional foreign key field is omitted', async () => {
+      await create('exercises', exerciseDefinition, exerciseKeys)
+      await create('workouts', workoutDefinition, workoutKeys)
+      const result = await insertEntity('workouts', [{}], {})
+      expect(result.workouts[0].exercise_id).toBeUndefined()
+    })
+  })
+})

package/src/data/queries/insertEntity.ts

@@ -0,0 +1,113 @@
+import { retrieveRecord } from '../utilities/retrieveRecord'
+import { updateIndex } from '../utilities/updateIndex'
+import { keyGenerate } from '../introspect/policies/keyGenerate'
+import { keyUnique } from '../introspect/policies/keyUnique'
+import { useDefault } from '../introspect/policies/useDefault'
+import { typeCheck } from '../introspect/policies/typeCheck'
+import { foreignKeyExists } from '../introspect/policies/foreignKeyExists'
+import { splitEntityProperty } from '../utilities/parsers/splitEntityProperty'
+import { recordDefinition } from '../introspect/definitions/record'
+
+const gulpConfig = require('js-build-tools/gulp.config')
+const { writeFile } = require('fs/promises')
+
+/**
+ * Build one entity's stored values from the raw input values, generating any autoGenerate
+ * fields and applying field defaults, then throw if a required field still has no value.
+ * @param entity
+ * @param record
+ * @param rowValues
+ */
+const buildEntityValues = async (entity: string, record: recordDefinition, rowValues: Object = {}): Promise<Object> => {
+  const entityValues = {}
+  for (const field of record.definition) {
+    let value = rowValues[field.name]
+    if (field.autoGenerate) {
+      value = await keyGenerate(entity, [field.name])
+    } else if (field.hasOwnProperty('default')) {
+      value = useDefault(field['default'], value)
+    }
+    if (typeof value === 'undefined') {
+      if (!field.optional) {
+        throw new Error(`Missing required field "${field.name}" for ${entity}`)
+      }
+      continue
+    }
+    entityValues[field.name] = typeCheck(field.type, value)
+  }
+  return entityValues
+}
+
+/**
+ * Reject the insert if it collides with an existing primary or unique single-field key value.
+ * @param entity
+ * @param record
+ * @param entityValues
+ */
+const assertUnique = async (entity: string, record: recordDefinition, entityValues: Object): Promise<void> => {
+  for (const key of record.keys) {
+    if ((key.type !== 'primary' && key.type !== 'unique') || key.fields.length !== 1) {
+      continue
+    }
+    const isUnique = await keyUnique(entity, key.fields, entityValues[key.fields[0]])
+    if (!isUnique) {
+      throw new Error(`Duplicate value for ${key.type} key "${key.fields[0]}" on ${entity}`)
+    }
+  }
+}
+
+/**
+ * Reject the insert if a single-field foreign key's value doesn't exist in the record it
+ * references.
+ * @param entity
+ * @param record
+ * @param entityValues
+ */
+const assertForeignKeys = async (entity: string, record: recordDefinition, entityValues: Object): Promise<void> => {
+  for (const key of record.keys) {
+    if (key.type !== 'foreign' || key.fields.length !== 1 || !key.references || key.references.length !== 1) {
+      continue
+    }
+    const field = key.fields[0]
+    if (!entityValues.hasOwnProperty(field)) {
+      continue
+    }
+    const { entity: refEntity, property: refProperty } = splitEntityProperty(key.references[0].fields[0])
+    const exists = await foreignKeyExists(refEntity, refProperty, entityValues[field])
+    if (!exists) {
+      throw new Error(`Foreign key "${field}" on ${entity} references a nonexistent ${refEntity}.${refProperty} = ${entityValues[field]}`)
+    }
+  }
+}
+
+export const insertEntity = async (entity: string = '', values: Object[] = [], dataSet: Object = {}): Promise<Object> => {
+  const databasePath = gulpConfig.get('databasePath', 'database/')
+  const record = await retrieveRecord(entity)
+  const primaryKey = record.keys.find(key => key.type === 'primary' && key.fields.length === 1)
+
+  if (!dataSet.hasOwnProperty(entity)) {
+    dataSet[entity] = []
+  }
+
+  for (const rowValues of values) {
+    const entityValues = await buildEntityValues(entity, record, rowValues)
+    await assertUnique(entity, record, entityValues)
+    await assertForeignKeys(entity, record, entityValues)
+
+    const fileName = `${primaryKey ? entityValues[primaryKey.fields[0]] : crypto.randomUUID()}.json`
+    await writeFile(`${databasePath}${record.path}/${fileName}`, JSON.stringify(entityValues, null, 2))
+
+    record.entries.push(fileName)
+    await writeFile(`${databasePath}${record.path}.json`, JSON.stringify(record, null, 2))
+
+    for (const key of record.keys) {
+      if (key.fields.length === 1) {
+        await updateIndex(record.path, key, entityValues[key.fields[0]], fileName)
+      }
+    }
+
+    dataSet[entity].push(entityValues)
+  }
+
+  return dataSet
+}

package/src/data/queries/joinEntity.test.js

@@ -0,0 +1,109 @@
+import { joinEntity } from './joinEntity'
+import { testHelpers } from 'js-build-tools/testHelpers'
+import { copyDatabase } from '../../../test-data/copyDatabase'
+
+const databasePath = 'join-entity-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(
+  () => testHelpers.beforeEach()
+    .then(() => copyDatabase(databasePath))
+)
+
+afterEach(testHelpers.afterEach)
+
+describe('joinEntity', () => {
+  test('it will find the joins', async () => {
+    const joinClauses = [
+      {
+        and: undefined,
+        propertyA: 'workouts.exercise_id',
+        comparator: '=',
+        propertyB: 'exercises._id'
+      },
+      {
+        and: 'and',
+        propertyA: 'exercises._id',
+        comparator: '=',
+        propertyB: 'exercise_muscle.exercise_id'
+      },
+      {
+        and: 'and',
+        propertyA: 'exercise_muscle.muscle_id',
+        comparator: '=',
+        propertyB: 'muscles._id'
+      }
+    ]
+    const result = await joinEntity('workouts', 'muscles', {}, joinClauses)
+    expect(result['workouts']).toEqual([
+      {
+        _id: 1,
+        exercise_id: 1,
+        weight_effort: '115lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 0,
+        date: '2024-06-02',
+        muscles: [{ _id: 11, name: 'Pectorals', alias: 'chest', image: '' }]
+      },
+      {
+        _id: 2,
+        exercise_id: 2,
+        weight_effort: '155lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 1,
+        date: '2024-06-02',
+        muscles: [{ _id: 8, name: 'Hamstrings', alias: 'abs', image: '' }]
+      },
+      {
+        _id: 3,
+        exercise_id: 3,
+        weight_effort: '155lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 2,
+        date: '2024-06-02',
+        muscles: [{ _id: 12, name: 'Quadriceps', alias: 'quads', image: '' }]
+      },
+      {
+        _id: 4,
+        exercise_id: 1,
+        weight_effort: '120lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 0,
+        date: '2024-06-05',
+        muscles: [{ _id: 11, name: 'Pectorals', alias: 'chest', image: '' }]
+      },
+      {
+        _id: 5,
+        exercise_id: 2,
+        weight_effort: '160lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 1,
+        date: '2024-06-05',
+        muscles: [{ _id: 8, name: 'Hamstrings', alias: 'abs', image: '' }]
+      },
+      {
+        _id: 6,
+        exercise_id: 3,
+        weight_effort: '160lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 2,
+        date: '2024-06-05',
+        muscles: [{ _id: 12, name: 'Quadriceps', alias: 'quads', image: '' }]
+      }
+    ])
+  })
+})

package/src/data/queries/joinEntity.ts

@@ -0,0 +1,20 @@
+import siFunciona from 'si-funciona'
+import { readEntity } from './readEntity'
+import { useJoinClause } from '../utilities/useJoinClause'
+
+export const joinEntity = async (entity, joinEntity, dataSets = {}, joinClauses = []) => {
+  dataSets = await readEntity(entity, dataSets)
+  if (!joinClauses.length) {
+    return dataSets
+  }
+  dataSets = await readEntity(joinEntity, dataSets)
+  for (const anEntity in dataSets[entity]) {
+    let filterJoins = siFunciona.cloneObject(dataSets)
+    filterJoins[entity] = [dataSets[entity][anEntity]]
+    for (let joinClause of joinClauses) {
+      filterJoins = await useJoinClause(joinClause, dataSets, filterJoins)
+    }
+    dataSets[entity][anEntity][joinEntity] = filterJoins[joinEntity]
+  }
+  return dataSets
+}

package/src/data/queries/mergeJoins.test.js

@@ -0,0 +1,174 @@
+import { mergeJoins } from './mergeJoins'
+import { testHelpers } from 'js-build-tools/testHelpers'
+import { copyDatabase } from '../../../test-data/copyDatabase'
+
+const databasePath = 'merge-joins-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(
+  () => testHelpers.beforeEach()
+    .then(() => copyDatabase(databasePath))
+)
+
+afterEach(testHelpers.afterEach)
+
+describe('mergeJoins', () => {
+  test('can combine several merges', async () => {
+    const results = await mergeJoins(
+      {},
+      [
+        {
+          propertyA: 'workouts.exercise_id',
+          propertyB: 'exercises._id'
+        },
+        {
+          propertyA: 'muscles._id',
+          propertyB: 'exercise_muscle.muscle_id'
+        }
+      ]
+    )
+    expect(results).toEqual({
+      workouts: [
+        {
+          _id: 1,
+          exercise_id: 1,
+          weight_effort: '115lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 0,
+          date: '2024-06-02',
+          'exercises.name': 'Bench Press',
+          'exercises.description': ''
+        },
+        {
+          _id: 2,
+          exercise_id: 2,
+          weight_effort: '155lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 1,
+          date: '2024-06-02',
+          'exercises.name': 'Deadlift',
+          'exercises.description': ''
+        },
+        {
+          _id: 3,
+          exercise_id: 3,
+          weight_effort: '155lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 2,
+          date: '2024-06-02',
+          'exercises.name': 'Squat',
+          'exercises.description': ''
+        },
+        {
+          _id: 4,
+          exercise_id: 1,
+          weight_effort: '120lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 0,
+          date: '2024-06-05',
+          'exercises.name': 'Bench Press',
+          'exercises.description': ''
+        },
+        {
+          _id: 5,
+          exercise_id: 2,
+          weight_effort: '160lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 1,
+          date: '2024-06-05',
+          'exercises.name': 'Deadlift',
+          'exercises.description': ''
+        },
+        {
+          _id: 6,
+          exercise_id: 3,
+          weight_effort: '160lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 2,
+          date: '2024-06-05',
+          'exercises.name': 'Squat',
+          'exercises.description': ''
+        }
+      ],
+      exercises: [
+        { _id: 1, name: 'Bench Press', description: '' },
+        { _id: 2, name: 'Deadlift', description: '' },
+        { _id: 3, name: 'Squat', description: '' }
+      ],
+      muscles: [
+        {
+          _id: 1,
+          name: 'Abdominal',
+          alias: 'abs',
+          image: '../img/abdominals.jpg'
+        },
+        { _id: 2, name: 'Biceps', alias: '', image: '' },
+        { _id: 3, name: 'Calves', alias: '', image: '' },
+        { _id: 4, name: 'Deltoids', alias: 'shoulders', image: '' },
+        {
+          _id: 5,
+          name: 'Erector Spinae',
+          alias: 'mid back, along spine',
+          image: ''
+        },
+        { _id: 6, name: 'Forearms', alias: 'abs', image: '' },
+        { _id: 7, name: 'Gluteus Maximus', alias: 'abs', image: '' },
+        {
+          _id: 8,
+          name: 'Hamstrings',
+          alias: 'abs',
+          image: '',
+          'exercise_muscle._id': 2,
+          'exercise_muscle.exercise_id': 2,
+          'exercise_muscle.muscle_rank': 100
+        },
+        {
+          _id: 9,
+          name: 'Latissimus Dorsi',
+          alias: 'outer lower back',
+          image: ''
+        },
+        { _id: 10, name: 'Obliques', alias: 'outer abs', image: '' },
+        {
+          _id: 11,
+          name: 'Pectorals',
+          alias: 'chest',
+          image: '',
+          'exercise_muscle._id': 1,
+          'exercise_muscle.exercise_id': 1,
+          'exercise_muscle.muscle_rank': 100
+        },
+        {
+          _id: 12,
+          name: 'Quadriceps',
+          alias: 'quads',
+          image: '',
+          'exercise_muscle._id': 3,
+          'exercise_muscle.exercise_id': 3,
+          'exercise_muscle.muscle_rank': 100
+        },
+        { _id: 13, name: 'Trapezius', alias: 'upper back', image: '' },
+        { _id: 14, name: 'Triceps', alias: '', image: '' }
+      ],
+      exercise_muscle: [
+        { _id: 1, exercise_id: 1, muscle_id: 11, muscle_rank: 100 },
+        { _id: 2, exercise_id: 2, muscle_id: 8, muscle_rank: 100 },
+        { _id: 3, exercise_id: 3, muscle_id: 12, muscle_rank: 100 }
+      ]
+    })
+  })
+})

package/src/data/queries/mergeJoins.ts

@@ -0,0 +1,13 @@
+import { mergeJoin } from '../utilities/mergeJoin'
+import { reconcileJoins } from '../utilities/reconcileJoins'
+
+export const mergeJoins = async (dataSet = {}, merges = []) => {
+  if (!merges.length) {
+    return dataSet
+  }
+  for (let merge of merges) {
+    dataSet = await mergeJoin(merge.propertyA, merge.propertyB, dataSet)
+    dataSet = reconcileJoins(merge.propertyA, merge.propertyB, dataSet)
+  }
+  return dataSet
+}