json-fs-query 0.1.0

package/src/data/introspect/drop.test.js

@@ -0,0 +1,77 @@
+import { create } from './create'
+import { drop } from './drop'
+import { insertEntity } from '../queries/insertEntity'
+import { testHelpers } from 'js-build-tools/testHelpers'
+
+const { writeFile, readFile } = require('fs/promises')
+
+const databasePath = 'drop-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+describe('drop', () => {
+  test('returns null for the reserved __RECORDS name, without touching anything', async () => {
+    expect(await drop('__RECORDS')).toBeNull()
+  })
+
+  test('removes every entry file listed on the record', async () => {
+    const recordName = 'foo'
+    await create(recordName, [], [])
+    // insertEntity is not implemented yet, so simulate a real entry directly: write the data file, and record it
+    // on the record's own entries list, exactly as a working insertEntity would.
+    const entryFile = 'entry1.json'
+    await writeFile(`${databasePath}${recordName}/${entryFile}`, JSON.stringify({ _id: 1 }))
+    const recordContent = JSON.parse(await readFile(`${databasePath}${recordName}.json`, 'utf8'))
+    recordContent.entries = [entryFile]
+    await writeFile(`${databasePath}${recordName}.json`, JSON.stringify(recordContent))
+
+    expect(testHelpers.fileExists(`${databasePath}${recordName}/${entryFile}`)).toBeTruthy()
+    await drop(recordName)
+    expect(testHelpers.fileExists(`${databasePath}${recordName}/${entryFile}`)).toBeFalsy()
+  })
+
+  test('deregisters the record from __RECORDS', async () => {
+    await create('foo', [], [])
+    await create('bar', [], [])
+    await drop('foo')
+    const registry = JSON.parse(await readFile(`${databasePath}__RECORDS.json`, 'utf8'))
+    expect(registry.entries).toEqual(['bar.json'])
+  })
+
+  test('removes the record\'s own definition file and data directory', async () => {
+    await create('foo', [], [])
+    expect(testHelpers.fileExists(`${databasePath}foo.json`)).toBeTruthy()
+    expect(testHelpers.fileExists(`${databasePath}foo`)).toBeTruthy()
+
+    await drop('foo')
+
+    expect(testHelpers.fileExists(`${databasePath}foo.json`)).toBeFalsy()
+    expect(testHelpers.fileExists(`${databasePath}foo`)).toBeFalsy()
+  })
+
+  test('removes the record\'s own index directory', async () => {
+    const definition = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: true }
+    ]
+    const keys = [
+      { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+    ]
+    await create('foo', definition, keys)
+    await insertEntity('foo', [{}], {})
+    expect(testHelpers.fileExists(`${databasePath}__indexes/foo/pk__id.json`)).toBeTruthy()
+
+    await drop('foo')
+
+    expect(testHelpers.fileExists(`${databasePath}__indexes/foo`)).toBeFalsy()
+  })
+
+  test('does not throw when the record has no index directory to remove', async () => {
+    await create('foo', [], [])
+    await expect(drop('foo')).resolves.not.toThrow()
+  })
+})

package/src/data/introspect/drop.ts

@@ -0,0 +1,20 @@
+import { retrieveFile } from '../utilities/retrieveFile'
+import { deregisterRecord } from '../utilities/deregisterRecord'
+
+const gulpConfig = require('js-build-tools/gulp.config')
+const { rm } = require('fs/promises')
+
+export const drop = async (record) => {
+  const databasePath = gulpConfig.get('databasePath', 'database/')
+  if (record === '__RECORDS') {
+    return null
+  }
+  const recordFile = await retrieveFile(`${record}.json`)
+  for (const file of recordFile.entries) {
+    await rm(`${databasePath}${recordFile.path}/${file}`)
+  }
+  await rm(`${databasePath}${recordFile.path}`, { recursive: true, force: true })
+  await rm(`${databasePath}__indexes/${recordFile.path}`, { recursive: true, force: true })
+  await rm(`${databasePath}${record}.json`)
+  await deregisterRecord(record)
+}

package/src/data/introspect/index.ts

@@ -0,0 +1,8 @@
+export { create } from './create'
+export { drop } from './drop'
+export { describe } from './describe'
+
+export { fieldProperties, fieldDefinition, fieldName } from './definitions/field'
+export { keyProperties, keyDefinition, indexType, indexLocation } from './definitions/key'
+export { referenceProperties, reference } from './definitions/keyReference'
+export { recordProperties, recordDefinition, recordPath, entityPath } from './definitions/record'

package/src/data/introspect/policies/foreignKeyExists.test.js

@@ -0,0 +1,44 @@
+import { create } from '../create'
+import { insertEntity } from '../../queries/insertEntity'
+import { foreignKeyExists } from './foreignKeyExists'
+import { testHelpers } from 'js-build-tools/testHelpers'
+
+const databasePath = 'foreign-key-exists-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+const definition = [
+  { name: '_id', type: 'string', optional: false, autoGenerate: true },
+  { name: 'name', type: 'string', optional: true, autoGenerate: false }
+]
+
+const keys = [
+  { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+]
+
+describe('foreignKeyExists', () => {
+  test('returns true when a record has the given property set to the given value', async () => {
+    await create('exercises', definition, keys)
+    const inserted = await insertEntity('exercises', [{ name: 'Bench Press' }], {})
+    const exists = await foreignKeyExists('exercises', '_id', inserted.exercises[0]._id)
+    expect(exists).toBe(true)
+  })
+
+  test('returns false when no record matches', async () => {
+    await create('exercises', definition, keys)
+    await insertEntity('exercises', [{ name: 'Bench Press' }], {})
+    const exists = await foreignKeyExists('exercises', '_id', 'does-not-exist')
+    expect(exists).toBe(false)
+  })
+
+  test('returns false for an entity with no entries', async () => {
+    await create('exercises', definition, keys)
+    const exists = await foreignKeyExists('exercises', '_id', 'anything')
+    expect(exists).toBe(false)
+  })
+})

package/src/data/introspect/policies/foreignKeyExists.ts

@@ -0,0 +1,15 @@
+import { retrieveRecords } from '../../utilities/retrieveRecords'
+import { where } from '../../queries/where'
+
+/**
+ * Check whether any record in the referenced entity has the given property set to the given
+ * value - the mirror image of keyUnique(), which checks that no record already has a value.
+ * @param entity
+ * @param property
+ * @param value
+ */
+export const foreignKeyExists = async (entity: string, property: string, value: any): Promise<boolean> => {
+  const records = await retrieveRecords(entity)
+  const found = where(records, { property, comparator: '=', value })
+  return found.length > 0
+}

package/src/data/introspect/policies/keyGenerate.ts

@@ -0,0 +1,11 @@
+import { keyUnique } from './keyUnique'
+
+export const keyGenerate = async (entity, references = []) => {
+  let uuid = null
+  let isUnique = false
+  while (!isUnique) {
+    uuid = crypto.randomUUID()
+    isUnique = await keyUnique(entity, references, uuid)
+  }
+  return uuid
+}

package/src/data/introspect/policies/keyUnique.ts

@@ -0,0 +1,11 @@
+import { retrieveRecords } from '../../utilities/retrieveRecords'
+import { where } from '../../queries/where'
+
+export const keyUnique = async (entity, references = [], checkKey = null) => {
+  if (references.length !== 1) {
+    return false
+  }
+  const records = await retrieveRecords(entity)
+  const found = where(records, { property: references[0], comparator: '=', value: checkKey })
+  return !found.length
+}

package/src/data/introspect/policies/typeCheck.test.js

@@ -0,0 +1,84 @@
+import { typeCheck } from './typeCheck'
+
+describe('typeCheck', () => {
+  test('passes an undefined value through unchanged, for optional fields', () => {
+    expect(typeCheck('string', undefined)).toBeUndefined()
+  })
+
+  test('does not constrain an unrecognized type name', () => {
+    expect(typeCheck('whatever', 42)).toBe(42)
+  })
+
+  describe('string', () => {
+    test('leaves an existing string alone', () => {
+      expect(typeCheck('string', 'foo')).toBe('foo')
+    })
+
+    test('coerces a non-string to a string', () => {
+      expect(typeCheck('string', 42)).toBe('42')
+    })
+  })
+
+  describe('int', () => {
+    test('leaves an existing integer alone', () => {
+      expect(typeCheck('int', 42)).toBe(42)
+    })
+
+    test('coerces a numeric string to a number', () => {
+      expect(typeCheck('int', '42')).toBe(42)
+    })
+
+    test('rejects a non-integer number', () => {
+      expect(() => typeCheck('int', 4.2)).toThrow('Value "4.2" is not a valid int')
+    })
+
+    test('rejects a value that cannot be coerced to a number', () => {
+      expect(() => typeCheck('int', 'not a number')).toThrow('Value "not a number" is not a valid int')
+    })
+  })
+
+  describe('boolean', () => {
+    test('leaves an existing boolean alone', () => {
+      expect(typeCheck('boolean', false)).toBe(false)
+    })
+
+    test('coerces the strings "true"/"false" to real booleans', () => {
+      expect(typeCheck('boolean', 'true')).toBe(true)
+      expect(typeCheck('boolean', 'false')).toBe(false)
+    })
+
+    test('rejects anything else', () => {
+      expect(() => typeCheck('boolean', 'yes')).toThrow('Value "yes" is not a valid boolean')
+    })
+  })
+
+  describe('object', () => {
+    test('accepts a plain object', () => {
+      const value = { a: 1 }
+      expect(typeCheck('object', value)).toBe(value)
+    })
+
+    test('rejects null', () => {
+      expect(() => typeCheck('object', null)).toThrow('Value "null" is not a valid object')
+    })
+
+    test('rejects an array', () => {
+      expect(() => typeCheck('object', [])).toThrow('is not a valid object')
+    })
+
+    test('rejects a primitive', () => {
+      expect(() => typeCheck('object', 'foo')).toThrow('is not a valid object')
+    })
+  })
+
+  describe('array', () => {
+    test('accepts an array', () => {
+      const value = [1, 2, 3]
+      expect(typeCheck('array', value)).toBe(value)
+    })
+
+    test('rejects a non-array', () => {
+      expect(() => typeCheck('array', { a: 1 })).toThrow('is not a valid array')
+    })
+  })
+})

package/src/data/introspect/policies/typeCheck.ts

@@ -0,0 +1,49 @@
+/**
+ * Validate (and where sensible, coerce) a value against a field's declared type. Runs only
+ * against values being written by insertEntity/updateEntity - it never re-validates data already
+ * on disk, so a field whose declared type doesn't match its already-stored values (which can
+ * happen, since nothing enforced this before) is left alone until something actually updates it.
+ * An unrecognized type name is treated as unconstrained rather than rejected, so a typo in a
+ * schema's field type can't silently block every write to that field.
+ * @param type
+ * @param value
+ */
+export const typeCheck = (type: string = 'string', value: any = undefined): any => {
+  if (typeof value === 'undefined') {
+    return value
+  }
+  switch (type) {
+    case 'string':
+      return typeof value === 'string' ? value : String(value)
+    case 'int': {
+      const coerced = Number(value)
+      if (Number.isNaN(coerced) || !Number.isInteger(coerced)) {
+        throw new Error(`Value "${value}" is not a valid int`)
+      }
+      return coerced
+    }
+    case 'boolean':
+      if (typeof value === 'boolean') {
+        return value
+      }
+      if (value === 'true') {
+        return true
+      }
+      if (value === 'false') {
+        return false
+      }
+      throw new Error(`Value "${value}" is not a valid boolean`)
+    case 'object':
+      if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+        throw new Error(`Value "${value}" is not a valid object`)
+      }
+      return value
+    case 'array':
+      if (!Array.isArray(value)) {
+        throw new Error(`Value "${value}" is not a valid array`)
+      }
+      return value
+    default:
+      return value
+  }
+}

package/src/data/introspect/policies/typeEnum.ts

@@ -0,0 +1,6 @@
+export const typeEnum = (values = [], value = '', optional = false) => {
+  if (values.includes(value)) {
+    return true
+  }
+  return typeof value === 'undefined' && optional
+}

package/src/data/introspect/policies/useDefault.ts

@@ -0,0 +1,6 @@
+export const useDefault = (defaultValue = '', value = undefined) => {
+  if (typeof value === 'undefined') {
+    return defaultValue
+  }
+  return value
+}

package/src/data/queries/deleteEntity.test.js

@@ -0,0 +1,86 @@
+import { create } from '../introspect/create'
+import { insertEntity } from './insertEntity'
+import { deleteEntity } from './deleteEntity'
+import { testHelpers } from 'js-build-tools/testHelpers'
+
+const { readFile } = require('fs/promises')
+
+const databasePath = 'delete-entity-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+const definition = [
+  { name: '_id', type: 'string', optional: false, autoGenerate: true },
+  { name: 'bar', type: 'string', optional: true, autoGenerate: false }
+]
+
+const keys = [
+  { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+]
+
+describe('deleteEntity', () => {
+  test('does nothing when dataSet has no rows to delete', async () => {
+    await create('foo', definition, keys)
+    const inserted = await insertEntity('foo', [{ bar: 'keep' }], {})
+    const result = await deleteEntity('foo', { foo: [] })
+    expect(result).toEqual({ foo: [] })
+    const recordFile = JSON.parse(await readFile(`${databasePath}foo.json`, 'utf8'))
+    expect(recordFile.entries).toEqual([`${inserted.foo[0]._id}.json`])
+  })
+
+  test('throws when the record has no single-field primary key', async () => {
+    await create('foo', definition, [])
+    await expect(deleteEntity('foo', { foo: [{ _id: '1' }] }))
+      .rejects.toThrow('Cannot delete from "foo" without a single-field primary key')
+  })
+
+  test('removes the matching entry file, entries[] entry, and index entry', async () => {
+    await create('foo', definition, keys)
+    const inserted = await insertEntity('foo', [{ bar: 'one' }, { bar: 'two' }], {})
+    const [keep, remove] = inserted.foo
+
+    await deleteEntity('foo', { foo: [remove] })
+
+    expect(testHelpers.fileExists(`${databasePath}foo/${remove._id}.json`)).toBeFalsy()
+    expect(testHelpers.fileExists(`${databasePath}foo/${keep._id}.json`)).toBeTruthy()
+
+    const recordFile = JSON.parse(await readFile(`${databasePath}foo.json`, 'utf8'))
+    expect(recordFile.entries).toEqual([`${keep._id}.json`])
+
+    const indexFile = JSON.parse(await readFile(`${databasePath}__indexes/foo/pk__id.json`, 'utf8'))
+    expect(indexFile).toEqual([{ value: keep._id, record: [`${keep._id}.json`] }])
+  })
+
+  test('finds the right file to delete even when a filename does not match the primary key', async () => {
+    await create('foo', definition, keys)
+    const inserted = await insertEntity('foo', [{ bar: 'one' }], {})
+    const entryFile = `${inserted.foo[0]._id}.json`
+    const { rename } = require('fs/promises')
+    await rename(`${databasePath}foo/${entryFile}`, `${databasePath}foo/custom-name.json`)
+    const recordFile = JSON.parse(await readFile(`${databasePath}foo.json`, 'utf8'))
+    recordFile.entries = ['custom-name.json']
+    const { writeFile } = require('fs/promises')
+    await writeFile(`${databasePath}foo.json`, JSON.stringify(recordFile))
+
+    await deleteEntity('foo', { foo: [inserted.foo[0]] })
+
+    expect(testHelpers.fileExists(`${databasePath}foo/custom-name.json`)).toBeFalsy()
+    const finalRecordFile = JSON.parse(await readFile(`${databasePath}foo.json`, 'utf8'))
+    expect(finalRecordFile.entries).toEqual([])
+  })
+
+  test('deletes multiple rows from one call', async () => {
+    await create('foo', definition, keys)
+    const inserted = await insertEntity('foo', [{ bar: 'one' }, { bar: 'two' }, { bar: 'three' }], {})
+
+    await deleteEntity('foo', { foo: [inserted.foo[0], inserted.foo[2]] })
+
+    const recordFile = JSON.parse(await readFile(`${databasePath}foo.json`, 'utf8'))
+    expect(recordFile.entries).toEqual([`${inserted.foo[1]._id}.json`])
+  })
+})

package/src/data/queries/deleteEntity.ts

@@ -0,0 +1,56 @@
+import { retrieveRecord } from '../utilities/retrieveRecord'
+import { retrieveFile } from '../utilities/retrieveFile'
+import { removeFromIndex } from '../utilities/removeFromIndex'
+import { recordDefinition } from '../introspect/definitions/record'
+
+const gulpConfig = require('js-build-tools/gulp.config')
+const { rm, writeFile } = require('fs/promises')
+
+/**
+ * A record object carries no memory of which entry file it was read from, and files are not
+ * guaranteed to be named after any field's value, so the only reliable way to find a record's
+ * file is to re-scan every entry and match it by its primary key value.
+ * @param entity
+ * @param record
+ * @param primaryField
+ * @param deletedValues
+ */
+const removeMatchingEntries = async (entity: string, record: recordDefinition, primaryField: string, deletedValues: Set<any>): Promise<string[]> => {
+  const databasePath = gulpConfig.get('databasePath', 'database/')
+  const remainingEntries = []
+  for (const fileName of record.entries) {
+    const data = await retrieveFile(`${record.path}/${fileName}`)
+    if (!deletedValues.has(data[primaryField])) {
+      remainingEntries.push(fileName)
+      continue
+    }
+    await rm(`${databasePath}${record.path}/${fileName}`)
+    for (const key of record.keys) {
+      if (key.fields.length === 1) {
+        await removeFromIndex(record.path, key, data[key.fields[0]], fileName)
+      }
+    }
+  }
+  return remainingEntries
+}
+
+export const deleteEntity = async (entity: string = '', dataSet: Object = {}): Promise<Object> => {
+  const databasePath = gulpConfig.get('databasePath', 'database/')
+  const record = await retrieveRecord(entity)
+  const primaryKey = record.keys.find(key => key.type === 'primary' && key.fields.length === 1)
+  if (!primaryKey) {
+    throw new Error(`Cannot delete from "${entity}" without a single-field primary key`)
+  }
+  const primaryField = primaryKey.fields[0]
+
+  const toDelete = dataSet[entity] || []
+  if (!toDelete.length) {
+    return dataSet
+  }
+  const deletedValues = new Set(toDelete.map(row => row[primaryField]))
+
+  record.entries = await removeMatchingEntries(entity, record, primaryField, deletedValues)
+  await writeFile(`${databasePath}${record.path}.json`, JSON.stringify(record, null, 2))
+
+  return dataSet
+}

package/src/data/queries/groupBy.test.js

@@ -0,0 +1,135 @@
+import { groupBy } from './groupBy'
+
+describe('groupBy', () => {
+  test('it groups by the given property', () => {
+    const dataSet = [
+      {
+        _id: 2,
+        exercise_id: 2,
+        weight_effort: '155lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 1,
+        date: '2024-06-02'
+      },
+      {
+        _id: 1,
+        exercise_id: 1,
+        weight_effort: '115lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 0,
+        date: '2024-06-02'
+      },
+      {
+        _id: 1,
+        exercise_id: 1,
+        weight_effort: '115lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 0,
+        date: '2024-06-05'
+      },
+      {
+        _id: 3,
+        exercise_id: 3,
+        weight_effort: '155lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 2,
+        date: '2024-06-02'
+      },
+      {
+        _id: 2,
+        exercise_id: 2,
+        weight_effort: '155lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 1,
+        date: '2024-06-05'
+      },
+      {
+        _id: 3,
+        exercise_id: 3,
+        weight_effort: '155lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 2,
+        date: '2024-06-05'
+      }
+    ]
+    const result = groupBy(dataSet, 'date')
+    expect(result).toEqual({
+      '2024-06-02': [
+        {
+          _id: 2,
+          exercise_id: 2,
+          weight_effort: '155lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 1,
+          date: '2024-06-02'
+        },
+        {
+          _id: 1,
+          exercise_id: 1,
+          weight_effort: '115lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 0,
+          date: '2024-06-02'
+        },
+        {
+          _id: 3,
+          exercise_id: 3,
+          weight_effort: '155lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 2,
+          date: '2024-06-02'
+        }
+      ],
+      '2024-06-05': [
+        {
+          _id: 1,
+          exercise_id: 1,
+          weight_effort: '115lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 0,
+          date: '2024-06-05'
+        },
+        {
+          _id: 2,
+          exercise_id: 2,
+          weight_effort: '155lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 1,
+          date: '2024-06-05'
+        },
+        {
+          _id: 3,
+          exercise_id: 3,
+          weight_effort: '155lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 2,
+          date: '2024-06-05'
+        }
+      ]
+    })
+  })
+})

package/src/data/queries/groupBy.ts

@@ -0,0 +1,16 @@
+export const groupBy = (dataSet = [], groupProperty = '') => {
+  if (!groupProperty) {
+    return dataSet
+  }
+  return dataSet.reduce(
+    (groupedData, data) => {
+      const property = data[groupProperty]
+      if (!groupedData.hasOwnProperty(property)) {
+        groupedData[property] = []
+      }
+      groupedData[property].push(data)
+      return groupedData
+    },
+    {}
+  )
+}