json-fs-query 0.1.0

package/src/data/utilities/runQuery.ts

@@ -0,0 +1,75 @@
+import { deleteEntity } from '../queries/deleteEntity'
+import { insertEntity } from '../queries/insertEntity'
+import { readEntity } from '../queries/readEntity'
+import { updateEntity } from '../queries/updateEntity'
+import { joinEntity } from '../queries/joinEntity'
+import { mergeJoins } from '../queries/mergeJoins'
+import { where } from '../queries/where'
+import { sortBy } from '../queries/sortBy'
+import { groupBy } from '../queries/groupBy'
+import { retrieveRecord } from './retrieveRecord'
+import { getConditionIndexedList } from './getConditionIndexedList'
+import { getJoinIndexedList } from './getJoinIndexedList'
+
+export const runQuery = async (parsed) => {
+  const recordData = {
+    [parsed.entity]: await retrieveRecord(parsed.entity)
+  }
+  if (typeof parsed.joinEntity !== 'undefined') {
+    recordData[parsed.joinEntity] = await retrieveRecord(parsed.joinEntity)
+  }
+
+  let dataSet: any = { [parsed.entity]: [] }
+
+  parsed = await getConditionIndexedList(parsed, recordData, dataSet)
+  parsed = await getJoinIndexedList(parsed, recordData, dataSet)
+
+  // delete/update act on whatever is already in dataSet[entity] - unlike read, which loads
+  // everything and lets the where-reduce below narrow it down afterward, they never get a second
+  // pass. So the matching rows have to be fully resolved (indexed conditions already are, via
+  // getConditionIndexedList above; anything left in parsed.conditions is not) before either runs.
+  if (parsed.command === 'delete' || parsed.command === 'update') {
+    dataSet = await readEntity(parsed.entity, dataSet)
+    dataSet[parsed.entity] = parsed.conditions.reduce(
+      (remainingData, condition) => where(remainingData, condition),
+      dataSet[parsed.entity]
+    )
+  }
+
+  switch (parsed.command) {
+    case 'delete':
+      dataSet = await deleteEntity(parsed.entity, dataSet)
+      break
+    case 'insert':
+      dataSet = await insertEntity(parsed.entity, parsed.insertValues, dataSet)
+      break
+    case 'read':
+      dataSet = await readEntity(parsed.entity, dataSet)
+      break
+    case 'update':
+      dataSet = await updateEntity(parsed.entity, parsed.updateValues, dataSet)
+      break
+    default:
+      throw new Error(`Unknown command: ${parsed.command}`)
+  }
+  dataSet = await joinEntity(parsed.entity, parsed.joinEntity, dataSet, parsed.joinClauses)
+  dataSet = await mergeJoins(dataSet, parsed.mergeJoins)
+
+  // delete/update already had parsed.conditions applied above, before they ran. Re-applying the
+  // same conditions here would be wrong, not just redundant, for update specifically: if the
+  // update just changed the very field a condition filters on, re-checking that condition against
+  // the now-changed data would filter the just-updated rows back out.
+  if (parsed.command !== 'delete' && parsed.command !== 'update') {
+    dataSet[parsed.entity] = parsed.conditions.reduce(
+      (remainingData, condition) => {
+        return where(remainingData, condition)
+      },
+      dataSet[parsed.entity]
+    )
+  }
+
+  dataSet[parsed.entity] = sortBy(dataSet[parsed.entity], parsed.sortClauses)
+  dataSet[parsed.entity] = groupBy(dataSet[parsed.entity], parsed.groupBy)
+
+  return dataSet
+}

package/src/data/utilities/updateIndex.test.js

@@ -0,0 +1,44 @@
+import { updateIndex } from './updateIndex'
+import { testHelpers } from 'js-build-tools/testHelpers'
+
+const databasePath = 'update-index-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+describe('updateIndex', () => {
+  test('does nothing for a key with no lookup', async () => {
+    const result = await updateIndex('foo', { type: 'multi', fields: ['a', 'b'], lookup: '' }, 'x', 'x.json')
+    expect(result).toBeNull()
+  })
+
+  test('creates a new index file the first time a value is indexed', async () => {
+    const key = { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+    const result = await updateIndex('foo', key, 1, '1.json')
+    expect(result).toEqual([{ value: 1, record: ['1.json'] }])
+    expect(testHelpers.fileExists(`${databasePath}__indexes/foo/pk__id.json`)).toBeTruthy()
+  })
+
+  test('appends a new value onto an existing index file', async () => {
+    const key = { type: 'index', fields: ['date'], lookup: 'index_date.json' }
+    await updateIndex('foo', key, '2024-06-02', '1.json')
+    const result = await updateIndex('foo', key, '2024-06-05', '2.json')
+    expect(result).toEqual([
+      { value: '2024-06-02', record: ['1.json'] },
+      { value: '2024-06-05', record: ['2.json'] }
+    ])
+  })
+
+  test('adds a matching file onto an existing value instead of duplicating the value', async () => {
+    const key = { type: 'index', fields: ['date'], lookup: 'index_date.json' }
+    await updateIndex('foo', key, '2024-06-02', '1.json')
+    const result = await updateIndex('foo', key, '2024-06-02', '2.json')
+    expect(result).toEqual([
+      { value: '2024-06-02', record: ['1.json', '2.json'] }
+    ])
+  })
+})

package/src/data/utilities/updateIndex.ts

@@ -0,0 +1,45 @@
+import { keyDefinition } from '../introspect/definitions/key'
+import { recordPath } from '../introspect/definitions/record'
+
+const gulpConfig = require('js-build-tools/gulp.config')
+const { readFile, writeFile, mkdir } = require('fs/promises')
+
+export type indexEntry = {
+  value: any
+  record: string[]
+}
+
+/**
+ * Add a single entry file to the index that backs one of a record's keys, creating the index
+ * file (and its __indexes/{record} directory) the first time a value is indexed. Keys with no
+ * lookup (multi-field keys are not indexed here) are left untouched.
+ * @param path
+ * @param key
+ * @param value
+ * @param fileName
+ */
+export const updateIndex = async (path: recordPath = '', key: keyDefinition = null, value: any = undefined, fileName: string = ''): Promise<indexEntry[] | null> => {
+  if (!key || !key.lookup) {
+    return null
+  }
+  const databasePath = gulpConfig.get('databasePath', 'database/')
+  const indexDir = `${databasePath}__indexes/${path}`
+  const indexPath = `${indexDir}/${key.lookup}`
+  let index: indexEntry[] = []
+  try {
+    index = JSON.parse(await readFile(indexPath, 'utf8'))
+  } catch (err) {
+    index = []
+  }
+  const existingEntry = index.find(entry => entry.value == value)
+  if (existingEntry) {
+    if (!existingEntry.record.includes(fileName)) {
+      existingEntry.record.push(fileName)
+    }
+  } else {
+    index.push({ value, record: [fileName] })
+  }
+  await mkdir(indexDir, { recursive: true })
+  await writeFile(indexPath, JSON.stringify(index, null, 2))
+  return index
+}

package/src/data/utilities/useJoinClause.test.js

@@ -0,0 +1,94 @@
+import { useJoinClause } from './useJoinClause'
+import { testHelpers } from 'js-build-tools/testHelpers'
+import { copyDatabase } from '../../../test-data/copyDatabase'
+
+const databasePath = 'use-join-clause-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(
+  () => testHelpers.beforeEach()
+    .then(() => copyDatabase(databasePath))
+)
+
+afterEach(testHelpers.afterEach)
+
+describe('useJoinClause', () => {
+  test('it will find the joins', async () => {
+    const result = await useJoinClause({
+      propertyA: 'workouts.exercise_id',
+      comparator: '=',
+      propertyB: 'exercises._id'
+    })
+    expect(result).toEqual({
+      workouts: [
+        {
+          _id: 1,
+          exercise_id: 1,
+          weight_effort: '115lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 0,
+          date: '2024-06-02'
+        },
+        {
+          _id: 4,
+          exercise_id: 1,
+          weight_effort: '120lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 0,
+          date: '2024-06-05'
+        },
+        {
+          _id: 2,
+          exercise_id: 2,
+          weight_effort: '155lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 1,
+          date: '2024-06-02'
+        },
+        {
+          _id: 5,
+          exercise_id: 2,
+          weight_effort: '160lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 1,
+          date: '2024-06-05'
+        },
+        {
+          _id: 3,
+          exercise_id: 3,
+          weight_effort: '155lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 2,
+          date: '2024-06-02'
+        },
+        {
+          _id: 6,
+          exercise_id: 3,
+          weight_effort: '160lbs',
+          reps: 10,
+          sets: 3,
+          note: '',
+          order: 2,
+          date: '2024-06-05'
+        }
+      ],
+      exercises: [
+        { _id: 1, name: 'Bench Press', description: '' },
+        { _id: 2, name: 'Deadlift', description: '' },
+        { _id: 3, name: 'Squat', description: '' }
+      ]
+    })
+  })
+})

package/src/data/utilities/useJoinClause.ts

@@ -0,0 +1,44 @@
+import { readEntity } from '../queries/readEntity'
+import { where } from '../queries/where'
+import { splitEntityProperty } from './parsers/splitEntityProperty'
+import siFunciona from 'si-funciona'
+
+export const useJoinClause = async (
+  { propertyA = null, comparator = '=', propertyB = null } = {},
+  dataSets = {},
+  filterJoins = {}
+) => {
+  const { entity: entityA, property: propA } = splitEntityProperty(propertyA)
+  const { entity: entityB, property: propB } = splitEntityProperty(propertyB)
+
+  if (!dataSets.hasOwnProperty(entityA)) {
+    dataSets = await readEntity(entityA, dataSets)
+  }
+  if (!dataSets.hasOwnProperty(entityB)) {
+    dataSets = await readEntity(entityB, dataSets)
+  }
+
+  if (!filterJoins.hasOwnProperty(entityA)) {
+    filterJoins[entityA] = siFunciona.cloneObject(dataSets[entityA])
+  }
+  if (!filterJoins.hasOwnProperty(entityB)) {
+    filterJoins[entityB] = siFunciona.cloneObject(dataSets[entityB])
+  }
+
+  filterJoins[entityA] = filterJoins[entityB].reduce(
+    (dataA, dataB) => siFunciona.mergeArrays(
+      dataA,
+      where(filterJoins[entityA], { property: propA, comparator: comparator, value: dataB[propB] })
+    ),
+    []
+  )
+
+  filterJoins[entityB] = filterJoins[entityA].reduce(
+    (dataB, dataA) => siFunciona.mergeArrays(
+      dataB,
+      where(filterJoins[entityB], { property: propB, comparator: comparator, value: dataA[propA] })
+    ),
+    []
+  )
+  return filterJoins
+}

package/src/global.d.ts

@@ -0,0 +1 @@
+declare function require (id: string): any

package/src/index.test.js

@@ -0,0 +1,52 @@
+import { query, create, drop, describe as describeRecord } from './index'
+import { testHelpers } from 'js-build-tools/testHelpers'
+
+const databasePath = 'public-api-database/'
+const gulpConfig = testHelpers.gulpConfig
+gulpConfig.set('databasePath', databasePath)
+testHelpers.setDefaults(databasePath)
+
+beforeEach(testHelpers.beforeEach)
+
+afterEach(testHelpers.afterEach)
+
+describe('the public package entry point', () => {
+  test('create, insert, read, update, delete, and drop all work through the exported functions', async () => {
+    const definition = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: true },
+      { name: 'note', type: 'string', optional: true, autoGenerate: false }
+    ]
+    const keys = [
+      { type: 'primary', fields: ['_id'], lookup: 'pk__id.json' }
+    ]
+
+    await create('foo', definition, keys)
+
+    const inserted = await query("insert foo values note = 'hello'")
+    expect(inserted[0]).toHaveLength(1)
+
+    const read = await query('read foo')
+    expect(read[0]).toHaveLength(1)
+    expect(read[0][0].note).toBe('hello')
+
+    const updated = await query("update foo set note = 'updated' where note = 'hello'")
+    expect(updated[0][0].note).toBe('updated')
+
+    const deleted = await query("delete foo where note = 'updated'")
+    expect(deleted[0]).toHaveLength(1)
+
+    const afterDelete = await query('read foo')
+    expect(afterDelete[0]).toEqual([])
+
+    await drop('foo')
+  })
+
+  test('describe reports a record\'s field definition', async () => {
+    const definition = [
+      { name: '_id', type: 'string', optional: false, autoGenerate: true }
+    ]
+    await create('bar', definition, [])
+    const result = await describeRecord('bar')
+    expect(result).toEqual(definition)
+  })
+})

package/src/index.ts

@@ -0,0 +1,17 @@
+export { query } from './data/queries/query'
+export { create, drop, describe } from './data/introspect'
+export {
+  fieldProperties,
+  fieldDefinition,
+  fieldName,
+  keyProperties,
+  keyDefinition,
+  indexType,
+  indexLocation,
+  referenceProperties,
+  reference,
+  recordProperties,
+  recordDefinition,
+  recordPath,
+  entityPath
+} from './data/introspect'

package/test-data/__RECORDS.json

@@ -0,0 +1,32 @@
+{
+  "path": "",
+  "definition": [
+    {
+      "name": "path",
+      "type": "string",
+      "optional": false
+    },
+    {
+      "name": "definition",
+      "type": "object",
+      "optional": false
+    },
+    {
+      "name": "keys",
+      "type": "array",
+      "optional": false
+    },
+    {
+      "name": "entries",
+      "type": "array",
+      "optional": false
+    }
+  ],
+  "keys":  [],
+  "entries": [
+    "exercise_muscle.json",
+    "exercises.json",
+    "muscles.json",
+    "workouts.json"
+  ]
+}

package/test-data/__indexes/exercise_muscle/fk_exercise_id.json

@@ -0,0 +1,14 @@
+[
+  {
+    "value": 1,
+    "record": ["bench_press_pectorals.json"]
+  },
+  {
+    "value": 2,
+    "record": ["deadlift_hamstrings.json"]
+  },
+  {
+    "value": 3,
+    "record": ["squat_quadriceps.json"]
+  }
+]

package/test-data/__indexes/exercise_muscle/fk_muscle_id.json

@@ -0,0 +1,14 @@
+[
+  {
+    "value": 8,
+    "record": ["deadlift_hamstrings.json"]
+  },
+  {
+    "value": 11,
+    "record": ["bench_press_pectorals.json"]
+  },
+  {
+    "value": 12,
+    "record": ["squat_quadriceps.json"]
+  }
+]

package/test-data/__indexes/exercise_muscle/pk__id.json

@@ -0,0 +1,14 @@
+[
+  {
+    "value": 1,
+    "record": ["bench_press_pectorals.json"]
+  },
+  {
+    "value": 2,
+    "record": ["deadlift_hamstrings.json"]
+  },
+  {
+    "value": 3,
+    "record": ["squat_quadriceps.json"]
+  }
+]

package/test-data/__indexes/exercises/pk__id.json

@@ -0,0 +1,14 @@
+[
+  {
+    "value": 1,
+    "record": ["bench_press.json"]
+  },
+  {
+    "value": 2,
+    "record": ["deadlift.json"]
+  },
+  {
+    "value": 3,
+    "record": ["squat.json"]
+  }
+]

package/test-data/__indexes/muscles/pk__id.json

@@ -0,0 +1,58 @@
+[
+  {
+    "value": 1,
+    "record": ["abdominal.json"]
+  },
+  {
+    "value": 2,
+    "record": ["biceps.json"]
+  },
+  {
+    "value": 3,
+    "record": ["calves.json"]
+  },
+  {
+    "value": 4,
+    "record": ["deltoids.json"]
+  },
+  {
+    "value": 5,
+    "record": ["erector_spinae.json"]
+  },
+  {
+    "value": 6,
+    "record": ["forearms.json"]
+  },
+  {
+    "value": 7,
+    "record": ["gluteas_maximus.json"]
+  },
+  {
+    "value": 8,
+    "record": ["hamstrings.json"]
+  },
+  {
+    "value": 9,
+    "record": ["latissimus_dorsi.json"]
+  },
+  {
+    "value": 10,
+    "record": ["obliques.json"]
+  },
+  {
+    "value": 11,
+    "record": ["pectorals.json"]
+  },
+  {
+    "value": 12,
+    "record": ["quadriceps.json"]
+  },
+  {
+    "value": 13,
+    "record": ["trapezius.json"]
+  },
+  {
+    "value": 14,
+    "record": ["triceps.json"]
+  }
+]

package/test-data/__indexes/workouts/fk_exercise_id.json

@@ -0,0 +1,23 @@
+[
+  {
+    "value": 1,
+    "record": [
+      "2024-06-02-001.json",
+      "2024-06-05-001.json"
+    ]
+  },
+  {
+    "value": 2,
+    "record": [
+      "2024-06-02-002.json",
+      "2024-06-05-002.json"
+    ]
+  },
+  {
+    "value": 3,
+    "record": [
+      "2024-06-02-003.json",
+      "2024-06-05-003.json"
+    ]
+  }
+]

package/test-data/__indexes/workouts/index_date.json

@@ -0,0 +1,18 @@
+[
+  {
+    "value": "2024-06-02",
+    "record": [
+      "2024-06-02-001.json",
+      "2024-06-02-002.json",
+      "2024-06-02-003.json"
+    ]
+  },
+  {
+    "value": "2024-06-05",
+    "record": [
+      "2024-06-05-001.json",
+      "2024-06-05-002.json",
+      "2024-06-05-003.json"
+    ]
+  }
+]

package/test-data/__indexes/workouts/pk__id.json

@@ -0,0 +1,26 @@
+[
+  {
+    "value": 1,
+    "record": ["2024-06-02-001.json"]
+  },
+  {
+    "value": 2,
+    "record": ["2024-06-02-001.json"]
+  },
+  {
+    "value": 3,
+    "record": ["2024-06-02-001.json"]
+  },
+  {
+    "value": 4,
+    "record": ["2024-06-05-001.json"]
+  },
+  {
+    "value": 5,
+    "record": ["2024-06-05-001.json"]
+  },
+  {
+    "value": 6,
+    "record": ["2024-06-05-001.json"]
+  }
+]

package/test-data/copyDatabase.ts

@@ -0,0 +1,64 @@
+// @ts-ignore
+import fs from 'fs/promises'
+
+export const copyDatabase = async (destination: string = 'test-database'): Promise<Array<string>> => {
+  const makeDirs: string[] = [
+    '__indexes',
+    '__indexes/exercise_muscle',
+    '__indexes/exercises',
+    '__indexes/muscles',
+    '__indexes/workouts',
+    'exercise_muscle',
+    'exercises',
+    'muscles',
+    'workouts',
+  ]
+  for (let dir of makeDirs) {
+    await fs.mkdir(`${destination}/${dir}`)
+  }
+  const copyFiles: string[] = [
+    '__RECORDS.json',
+    'exercise_muscle.json',
+    'exercise_muscle/bench_press_pectorals.json',
+    'exercise_muscle/deadlift_hamstrings.json',
+    'exercise_muscle/squat_quadriceps.json',
+    '__indexes/exercise_muscle/fk_exercise_id.json',
+    '__indexes/exercise_muscle/fk_muscle_id.json',
+    '__indexes/exercise_muscle/pk__id.json',
+    'exercises.json',
+    'exercises/bench_press.json',
+    'exercises/deadlift.json',
+    'exercises/squat.json',
+    '__indexes/exercises/pk__id.json',
+    'muscles.json',
+    'muscles/abdominal.json',
+    'muscles/biceps.json',
+    'muscles/calves.json',
+    'muscles/deltoids.json',
+    'muscles/erector_spinae.json',
+    'muscles/forearms.json',
+    'muscles/gluteus_maximus.json',
+    'muscles/hamstrings.json',
+    'muscles/latissimus_dorsi.json',
+    'muscles/obliques.json',
+    'muscles/pectorals.json',
+    'muscles/quadriceps.json',
+    'muscles/trapezius.json',
+    'muscles/triceps.json',
+    '__indexes/muscles/pk__id.json',
+    'workouts.json',
+    'workouts/2024-06-02-001.json',
+    'workouts/2024-06-02-002.json',
+    'workouts/2024-06-02-003.json',
+    'workouts/2024-06-05-001.json',
+    'workouts/2024-06-05-002.json',
+    'workouts/2024-06-05-003.json',
+    '__indexes/workouts/fk_exercise_id.json',
+    '__indexes/workouts/index_date.json',
+    '__indexes/workouts/pk__id.json',
+  ]
+  for (let file of copyFiles) {
+    await fs.copyFile(`test-data/${file}`, `${destination}/${file}`)
+  }
+  return copyFiles
+}

package/test-data/exercise_muscle/bench_press_pectorals.json

@@ -0,0 +1,6 @@
+{
+  "_id": 1,
+  "exercise_id": 1,
+  "muscle_id": 11,
+  "muscle_rank": 100
+}

package/test-data/exercise_muscle/deadlift_hamstrings.json

@@ -0,0 +1,6 @@
+{
+  "_id": 2,
+  "exercise_id": 2,
+  "muscle_id": 8,
+  "muscle_rank": 100
+}