json-fs-query 0.1.0

package/src/data/utilities/registerRecord.test.js

@@ -0,0 +1,44 @@
+import { registerRecord } from './registerRecord'
+import { testHelpers } from 'js-build-tools/testHelpers'
+
+const { readFile } = require('fs/promises')
+
+const databasePath = 'register-record-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+describe('registerRecord', () => {
+  test('creates a fresh registry the first time a record is registered', async () => {
+    const result = await registerRecord('foo')
+    expect(result).toEqual({
+      path: '',
+      definition: [
+        { name: 'path', type: 'string', optional: false, autoGenerate: false },
+        { name: 'definition', type: 'object', optional: false, autoGenerate: false },
+        { name: 'keys', type: 'array', optional: false, autoGenerate: false },
+        { name: 'entries', type: 'array', optional: false, autoGenerate: false }
+      ],
+      keys: [],
+      entries: ['foo.json']
+    })
+    const onDisk = JSON.parse(await readFile(`${databasePath}__RECORDS.json`, 'utf8'))
+    expect(onDisk).toEqual(result)
+  })
+
+  test('appends onto an existing registry without disturbing other entries', async () => {
+    await registerRecord('foo')
+    const result = await registerRecord('bar')
+    expect(result.entries).toEqual(['foo.json', 'bar.json'])
+  })
+
+  test('registering the same record twice does not duplicate its entry', async () => {
+    await registerRecord('foo')
+    const result = await registerRecord('foo')
+    expect(result.entries).toEqual(['foo.json'])
+  })
+})

package/src/data/utilities/registerRecord.ts

@@ -0,0 +1,44 @@
+import { recordDefinition } from '../introspect/definitions/record'
+
+const gulpConfig = require('js-build-tools/gulp.config')
+const { readFile, writeFile } = require('fs/promises')
+
+/**
+ * __RECORDS is itself an ordinary record - path/definition/keys/entries - whose entries are the
+ * filenames of every other record that has been created. This is the schema it's given the first
+ * time a record is registered and no __RECORDS.json exists yet, matching what a hand-authored
+ * database (see test-data/__RECORDS.json) already expects to find.
+ */
+const defaultRegistry = (): recordDefinition => ({
+  path: '',
+  definition: [
+    { name: 'path', type: 'string', optional: false, autoGenerate: false },
+    { name: 'definition', type: 'object', optional: false, autoGenerate: false },
+    { name: 'keys', type: 'array', optional: false, autoGenerate: false },
+    { name: 'entries', type: 'array', optional: false, autoGenerate: false }
+  ],
+  keys: [],
+  entries: []
+})
+
+/**
+ * Add a record's filename to the __RECORDS registry, creating the registry file the first time
+ * a record is created if it doesn't already exist.
+ * @param recordName
+ */
+export const registerRecord = async (recordName: string): Promise<recordDefinition> => {
+  const databasePath = gulpConfig.get('databasePath', 'database/')
+  const registryPath = `${databasePath}__RECORDS.json`
+  let registry: recordDefinition
+  try {
+    registry = JSON.parse(await readFile(registryPath, 'utf8'))
+  } catch (err) {
+    registry = defaultRegistry()
+  }
+  const fileName = `${recordName}.json`
+  if (!registry.entries.includes(fileName)) {
+    registry.entries.push(fileName)
+  }
+  await writeFile(registryPath, JSON.stringify(registry, null, 2))
+  return registry
+}

package/src/data/utilities/removeFromIndex.test.js

@@ -0,0 +1,44 @@
+import { updateIndex } from './updateIndex'
+import { removeFromIndex } from './removeFromIndex'
+import { testHelpers } from 'js-build-tools/testHelpers'
+
+const { readFile } = require('fs/promises')
+
+const databasePath = 'remove-from-index-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+describe('removeFromIndex', () => {
+  test('does nothing for a key with no lookup', async () => {
+    const result = await removeFromIndex('foo', { type: 'multi', fields: ['a', 'b'], lookup: '' }, 'x', 'x.json')
+    expect(result).toBeNull()
+  })
+
+  test('does nothing when the index file does not exist yet', async () => {
+    const key = { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+    const result = await removeFromIndex('foo', key, 1, '1.json')
+    expect(result).toBeNull()
+  })
+
+  test('removes a file from its value, keeping other files on that value', async () => {
+    const key = { type: 'index', fields: ['date'], lookup: 'index_date.json' }
+    await updateIndex('foo', key, '2024-06-02', '1.json')
+    await updateIndex('foo', key, '2024-06-02', '2.json')
+    const result = await removeFromIndex('foo', key, '2024-06-02', '1.json')
+    expect(result).toEqual([{ value: '2024-06-02', record: ['2.json'] }])
+  })
+
+  test('drops the whole value once its last file is removed', async () => {
+    const key = { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+    await updateIndex('foo', key, 1, '1.json')
+    const result = await removeFromIndex('foo', key, 1, '1.json')
+    expect(result).toEqual([])
+    const onDisk = JSON.parse(await readFile(`${databasePath}__indexes/foo/pk__id.json`, 'utf8'))
+    expect(onDisk).toEqual([])
+  })
+})

package/src/data/utilities/removeFromIndex.ts

@@ -0,0 +1,34 @@
+import { keyDefinition } from '../introspect/definitions/key'
+import { recordPath } from '../introspect/definitions/record'
+import { indexEntry } from './updateIndex'
+
+const gulpConfig = require('js-build-tools/gulp.config')
+const { readFile, writeFile } = require('fs/promises')
+
+/**
+ * Remove a single entry file from the index that backs one of a record's keys, dropping a value
+ * entirely once its last file is removed. A key with no lookup, or an index file that doesn't
+ * exist yet, is left untouched.
+ * @param path
+ * @param key
+ * @param value
+ * @param fileName
+ */
+export const removeFromIndex = async (path: recordPath = '', key: keyDefinition = null, value: any = undefined, fileName: string = ''): Promise<indexEntry[] | null> => {
+  if (!key || !key.lookup) {
+    return null
+  }
+  const databasePath = gulpConfig.get('databasePath', 'database/')
+  const indexPath = `${databasePath}__indexes/${path}/${key.lookup}`
+  let index: indexEntry[] = []
+  try {
+    index = JSON.parse(await readFile(indexPath, 'utf8'))
+  } catch (err) {
+    return null
+  }
+  index = index
+    .map(entry => entry.value == value ? { value: entry.value, record: entry.record.filter(record => record !== fileName) } : entry)
+    .filter(entry => entry.record.length)
+  await writeFile(indexPath, JSON.stringify(index, null, 2))
+  return index
+}

package/src/data/utilities/restrictIndexedList.ts

@@ -0,0 +1,38 @@
+import { splitEntityProperty } from './parsers/splitEntityProperty'
+
+export const restrictIndexedList = (joinClause, filterJoins, indexedClone) => {
+  const { entity: entityA, property: propertyA } = splitEntityProperty(joinClause.propertyA)
+  const { entity: entityB, property: propertyB } = splitEntityProperty(joinClause.propertyB)
+  if (filterJoins.hasOwnProperty(entityA)) {
+    const existingEntities = filterJoins[entityA]
+    if (!indexedClone.hasOwnProperty(entityA)) {
+      indexedClone[entityA] = {}
+    }
+    if (!indexedClone[entityA].hasOwnProperty(propertyA)) {
+      indexedClone[entityA][propertyA] = {}
+    }
+    indexedClone[entityA][propertyA] = [
+      {
+        value: existingEntities[0][propertyA],
+        record: existingEntities
+      }
+    ]
+  }
+  if (filterJoins.hasOwnProperty(entityB)) {
+    const existingEntities = filterJoins[entityB]
+    if (!indexedClone.hasOwnProperty(entityB)) {
+      indexedClone[entityB] = {}
+    }
+    if (!indexedClone[entityB].hasOwnProperty(propertyB)) {
+      indexedClone[entityB][propertyB] = {}
+    }
+    indexedClone[entityB][propertyB] = [
+      {
+        value: existingEntities[0][propertyB],
+        record: existingEntities
+      }
+    ]
+  }
+
+  return indexedClone
+}

package/src/data/utilities/retrieveFile.test.js

@@ -0,0 +1,49 @@
+import { retrieveFile } from './retrieveFile'
+import { testHelpers } from 'js-build-tools/testHelpers'
+import fs from 'fs'
+
+const databasePath = 'retrieve-file-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+describe('retrieveFile', () => {
+  test('should get the file if found', async () => {
+    const recordsContent = {
+      path: '',
+      definition: [
+        {
+          name: 'path',
+          type: 'string',
+          optional: false,
+          keys: []
+        },
+        {
+          name: 'definition',
+          type: 'object',
+          optional: false,
+          keys: []
+        },
+        {
+          name: 'entries',
+          type: 'array',
+          optional: false,
+          keys: []
+        },
+      ],
+      entries: [
+        'exercise_muscle.json',
+        'exercises.json',
+        'muscles.json',
+        'workouts.json',
+      ]
+    }
+    const fileName = '__RECORDS.json'
+    fs.writeFileSync(`${databasePath}${fileName}`, JSON.stringify(recordsContent))
+    expect(await retrieveFile(fileName)).toEqual(recordsContent)
+  })
+})

package/src/data/utilities/retrieveFile.ts

@@ -0,0 +1,19 @@
+import * as jsEnv from 'browser-or-node'
+import * as gulpConfig from 'js-build-tools/gulp.config'
+
+export const retrieveFile = async (path) => {
+  let retrieveFn = url => fetch(url).then(res => res.json())
+  let relativePath = jsEnv.isBrowser ? gulpConfig.get('relativePath', '') : ''
+  if (jsEnv.isNode) {
+    const { readFile } = require('fs/promises')
+    retrieveFn = url => readFile(url).then(res => JSON.parse(res))
+  }
+  const databasePath = gulpConfig.get('databasePath', 'database/')
+  const fetchUrl = `${relativePath}${databasePath}${path}`
+  const recordInfo = await retrieveFn(fetchUrl)
+    .catch(err => console.error(err))
+  if (!recordInfo) {
+    throw new Error(`Could not read ${path}`)
+  }
+  return recordInfo
+}

package/src/data/utilities/retrieveIndexes.ts

@@ -0,0 +1,17 @@
+import { retrieveFile } from './retrieveFile'
+import { retrieveRecord } from './retrieveRecord'
+
+export const retrieveIndexes = async (entity = '', properties = [], recordData = {}) => {
+  if (!recordData.hasOwnProperty(entity)) {
+    recordData[entity] = await retrieveRecord(entity)
+  }
+  const indexSet = {}
+  for (let property of properties) {
+    for (let key of recordData[entity].keys) {
+      if (key.fields.includes(property)) {
+        indexSet[property] = await retrieveFile(`__indexes/${entity}/${key.lookup}`)
+      }
+    }
+  }
+  return indexSet
+}

package/src/data/utilities/retrieveRecord.ts

@@ -0,0 +1,3 @@
+import { retrieveFile } from './retrieveFile'
+
+export const retrieveRecord = async (entity = '') => await retrieveFile(`${entity}.json`)

package/src/data/utilities/retrieveRecords.test.js

@@ -0,0 +1,223 @@
+import { retrieveRecords } from './retrieveRecords'
+import { testHelpers } from 'js-build-tools/testHelpers'
+import { copyDatabase } from '../../../test-data/copyDatabase'
+
+const databasePath = 'retrieve-records-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(
+  () => testHelpers.beforeEach()
+    .then(() => copyDatabase(databasePath))
+)
+
+afterEach(testHelpers.afterEach)
+
+describe('retrieveRecords', () => {
+  test('retrieve main RECORDS file', async () => {
+    expect(await retrieveRecords('__RECORDS')).toEqual([
+      {
+        path: 'exercise_muscle',
+        definition: [
+          {
+            name: '_id',
+            type: 'string',
+            optional: false
+          },
+          {
+            name: 'exercise_id',
+            type: 'string',
+            optional: false
+          },
+          {
+            name: 'muscle_id',
+            type: 'string',
+            optional: false
+          },
+          { name: 'muscle_rank', type: 'int', optional: false }
+        ],
+        keys: [
+          { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' },
+          {
+            type: 'foreign', fields: ['exercise_id'], lookup: 'fk_exercise_id.json', references: [
+              {
+                'fields': ['exercises._id'],
+                'lookup': 'exercises.json'
+              }
+            ]
+          },
+          {
+            type: 'foreign', fields: ['muscle_id'], lookup: 'fk_muscle_id.json', references: [
+              {
+                'fields': ['muscles._id'],
+                'lookup': 'muscles.json'
+              }
+            ]
+          }
+        ],
+        entries: [
+          'bench_press_pectorals.json',
+          'deadlift_hamstrings.json',
+          'squat_quadriceps.json'
+        ]
+      },
+      {
+        path: 'exercises',
+        definition: [
+          {
+            name: '_id',
+            type: 'string',
+            optional: false
+          },
+          { name: 'name', type: 'string', optional: false },
+          { name: 'description', type: 'string', optional: true }
+        ],
+        keys: [
+          { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' },
+        ],
+        entries: ['bench_press.json', 'deadlift.json', 'squat.json']
+      },
+      {
+        path: 'muscles',
+        definition: [
+          {
+            name: '_id',
+            type: 'string',
+            optional: false
+          },
+          { name: 'name', type: 'string', optional: false },
+          { name: 'alias', type: 'string', optional: true },
+          { name: 'image', type: 'string', optional: true }
+        ],
+        keys: [
+          { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' },
+        ],
+        entries: [
+          'abdominal.json',
+          'biceps.json',
+          'calves.json',
+          'deltoids.json',
+          'erector_spinae.json',
+          'forearms.json',
+          'gluteus_maximus.json',
+          'hamstrings.json',
+          'latissimus_dorsi.json',
+          'obliques.json',
+          'pectorals.json',
+          'quadriceps.json',
+          'trapezius.json',
+          'triceps.json'
+        ]
+      },
+      {
+        path: 'workouts',
+        definition: [
+          {
+            name: '_id',
+            type: 'string',
+            optional: false
+          },
+          {
+            name: 'exercise_id',
+            type: 'string',
+            optional: false
+          },
+          {
+            name: 'weight_effort',
+            type: 'string',
+            optional: true
+          },
+          { name: 'reps', type: 'int', optional: true },
+          { name: 'sets', type: 'int', optional: true },
+          { name: 'note', type: 'string', optional: true },
+          { name: 'order', type: 'int', optional: false },
+          { name: 'date', type: 'string', optional: false }
+        ],
+        keys: [
+          { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' },
+          { type: 'index', fields: ['date'], lookup: 'index_date.json' },
+          {
+            type: 'foreign',
+            fields: ['exercise_id'],
+            lookup: 'fk_exercise_id.json',
+            references: [{ fields: ['exercises._id'], lookup: 'exercises.json' }]
+          }
+        ],
+        entries: [
+          '2024-06-02-001.json',
+          '2024-06-02-002.json',
+          '2024-06-02-003.json',
+          '2024-06-05-001.json',
+          '2024-06-05-002.json',
+          '2024-06-05-003.json'
+        ]
+      }
+    ])
+  })
+
+  test('retrieve workouts entries', async () => {
+    expect(await retrieveRecords('workouts')).toEqual([
+      {
+        _id: 1,
+        exercise_id: 1,
+        weight_effort: '115lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 0,
+        date: '2024-06-02'
+      },
+      {
+        _id: 2,
+        exercise_id: 2,
+        weight_effort: '155lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 1,
+        date: '2024-06-02'
+      },
+      {
+        _id: 3,
+        exercise_id: 3,
+        weight_effort: '155lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 2,
+        date: '2024-06-02'
+      },
+      {
+        _id: 4,
+        exercise_id: 1,
+        weight_effort: '120lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 0,
+        date: '2024-06-05'
+      },
+      {
+        _id: 5,
+        exercise_id: 2,
+        weight_effort: '160lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 1,
+        date: '2024-06-05'
+      },
+      {
+        _id: 6,
+        exercise_id: 3,
+        weight_effort: '160lbs',
+        reps: 10,
+        sets: 3,
+        note: '',
+        order: 2,
+        date: '2024-06-05'
+      }
+    ])
+  })
+})

package/src/data/utilities/retrieveRecords.ts

@@ -0,0 +1,16 @@
+import { retrieveFile } from './retrieveFile'
+import { retrieveRecord } from './retrieveRecord'
+
+export const retrieveRecords = async (entity = '') => {
+  const recordFile = await retrieveRecord(entity)
+  const dataSet = []
+  for (let file of recordFile.entries) {
+    const data = await retrieveFile(`${recordFile.path}/${file}`)
+      .catch(err => console.error(err))
+    if (!data) {
+      throw new Error(`Could not read entity ${file}`)
+    }
+    dataSet.push(data)
+  }
+  return dataSet
+}

package/src/data/utilities/runQuery.test.js

@@ -0,0 +1,88 @@
+import { create } from '../introspect/create'
+import { insertEntity } from '../queries/insertEntity'
+import { runQuery } from './runQuery'
+import { testHelpers } from 'js-build-tools/testHelpers'
+
+const databasePath = 'run-query-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+const definition = [
+  { name: '_id', type: 'string', optional: false, autoGenerate: true },
+  { name: 'bar', type: 'string', optional: false, autoGenerate: false }
+]
+
+const keys = [
+  { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+]
+
+const baseParsed = (overrides) => Object.assign({
+  entity: 'foo',
+  joinEntity: undefined,
+  conditions: [],
+  joinClauses: [],
+  mergeJoins: [],
+  sortClauses: [],
+  groupBy: undefined
+}, overrides)
+
+describe('runQuery', () => {
+  describe('delete', () => {
+    test('deletes only the rows matching a non-indexed condition', async () => {
+      await create('foo', definition, keys)
+      await insertEntity('foo', [{ bar: 'keep' }, { bar: 'remove' }, { bar: 'keep' }], {})
+
+      const result = await runQuery(baseParsed({
+        command: 'delete',
+        conditions: [{ property: 'bar', comparator: '=', value: 'remove' }]
+      }))
+
+      expect(result.foo.map(row => row.bar)).toEqual(['remove'])
+      const remaining = await runQuery(baseParsed({ command: 'read' }))
+      expect(remaining.foo.map(row => row.bar).sort()).toEqual(['keep', 'keep'])
+    })
+
+    test('with no conditions, deletes every row', async () => {
+      await create('foo', definition, keys)
+      await insertEntity('foo', [{ bar: 'one' }, { bar: 'two' }], {})
+
+      await runQuery(baseParsed({ command: 'delete' }))
+
+      const remaining = await runQuery(baseParsed({ command: 'read' }))
+      expect(remaining.foo).toEqual([])
+    })
+  })
+
+  describe('update', () => {
+    test('updates only the rows matching a non-indexed condition', async () => {
+      await create('foo', definition, keys)
+      await insertEntity('foo', [{ bar: 'change-me' }, { bar: 'leave-me' }], {})
+
+      const result = await runQuery(baseParsed({
+        command: 'update',
+        conditions: [{ property: 'bar', comparator: '=', value: 'change-me' }],
+        updateValues: { bar: 'changed' }
+      }))
+
+      expect(result.foo).toHaveLength(1)
+      expect(result.foo[0].bar).toBe('changed')
+      const remaining = await runQuery(baseParsed({ command: 'read' }))
+      expect(remaining.foo.map(row => row.bar).sort()).toEqual(['changed', 'leave-me'])
+    })
+
+    test('with no conditions, updates every row', async () => {
+      await create('foo', definition, keys)
+      await insertEntity('foo', [{ bar: 'one' }, { bar: 'two' }], {})
+
+      await runQuery(baseParsed({ command: 'update', updateValues: { bar: 'all-changed' } }))
+
+      const remaining = await runQuery(baseParsed({ command: 'read' }))
+      expect(remaining.foo.map(row => row.bar)).toEqual(['all-changed', 'all-changed'])
+    })
+  })
+})