json-fs-query 3.1.4 → 3.3.0

package/dist/data/queries/insertEntity.js

@@ -4,14 +4,44 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.insertEntity = void 0;
+require("core-js/modules/es.array.includes.js");
 require("core-js/modules/es.json.stringify.js");
 require("core-js/modules/esnext.iterator.constructor.js");
+require("core-js/modules/esnext.iterator.every.js");
 require("core-js/modules/esnext.iterator.find.js");
+require("core-js/modules/esnext.iterator.for-each.js");
 require("core-js/modules/esnext.iterator.map.js");
 require("core-js/modules/esnext.iterator.some.js");
+require("core-js/modules/esnext.map.delete-all.js");
+require("core-js/modules/esnext.map.every.js");
+require("core-js/modules/esnext.map.filter.js");
+require("core-js/modules/esnext.map.find.js");
+require("core-js/modules/esnext.map.find-key.js");
+require("core-js/modules/esnext.map.includes.js");
+require("core-js/modules/esnext.map.key-of.js");
+require("core-js/modules/esnext.map.map-keys.js");
+require("core-js/modules/esnext.map.map-values.js");
+require("core-js/modules/esnext.map.merge.js");
+require("core-js/modules/esnext.map.reduce.js");
+require("core-js/modules/esnext.map.some.js");
+require("core-js/modules/esnext.map.update.js");
+require("core-js/modules/esnext.set.add-all.js");
+require("core-js/modules/esnext.set.delete-all.js");
+require("core-js/modules/esnext.set.difference.js");
+require("core-js/modules/esnext.set.every.js");
+require("core-js/modules/esnext.set.filter.js");
+require("core-js/modules/esnext.set.find.js");
+require("core-js/modules/esnext.set.intersection.js");
+require("core-js/modules/esnext.set.is-disjoint-from.js");
+require("core-js/modules/esnext.set.is-subset-of.js");
+require("core-js/modules/esnext.set.is-superset-of.js");
+require("core-js/modules/esnext.set.join.js");
+require("core-js/modules/esnext.set.map.js");
+require("core-js/modules/esnext.set.reduce.js");
+require("core-js/modules/esnext.set.some.js");
+require("core-js/modules/esnext.set.symmetric-difference.js");
+require("core-js/modules/esnext.set.union.js");
 var _retrieveRecord = require("../utilities/retrieveRecord");
-var _updateIndex = require("../utilities/updateIndex");
-var _keyGenerate = require("../introspect/policies/keyGenerate");
 var _keyUnique = require("../introspect/policies/keyUnique");
 var _useDefault = require("../introspect/policies/useDefault");
 var _typeCheck = require("../introspect/policies/typeCheck");
@@ -19,30 +49,290 @@ var _foreignKeyExists = require("../introspect/policies/foreignKeyExists");
 var _splitEntityProperty = require("../utilities/parsers/splitEntityProperty");
 var _computeKeyValue = require("../utilities/computeKeyValue");
 var _fieldType = require("../utilities/fieldType");
-var _config = require("../utilities/config");
-const {
-  writeFile,
-  mkdir
-} = require('fs/promises');
+var _compareAs = require("../utilities/compareAs");
+var _retrieveRecords = require("../utilities/retrieveRecords");
+var _isSingleFieldKey = require("../utilities/isSingleFieldKey");
+var _indexBuffer = require("../utilities/indexBuffer");
+var _storage = require("../storage");
+var _packedTable = require("../utilities/packedTable");
+var _knownFields = require("../utilities/knownFields");
+/**
+ * The rows of a record that hold each combination of values for some fields, compared field by field by type, as
+ * where's = compares them: built from the rows in storage once per statement, for a key no index can answer (several
+ * fields, or no index file), instead of looking at every row in storage once per inserted row.
+ */
+class RowValues {
+  constructor(record, fields) {
+    this.seen = new Set();
+    this.fields = fields;
+    this.types = fields.map(field => (0, _fieldType.fieldType)(record, field));
+  }
+  /**
+   * Whether these fields' values can be compared this way: every field of a comparable type (see compareAs).
+   * @param record
+   * @param fields
+   */
+  static usable(record, fields) {
+    return fields.every(field => _compareAs.comparableTypes.includes((0, _fieldType.fieldType)(record, field)));
+  }
+  /**
+   * The values' key, or undefined when one is null or missing (where's = never matches those) or compareAs leaves it an
+   * object (compared by identity, so not by a key).
+   * @param row
+   */
+  keyOf(row) {
+    const typed = [];
+    for (let position = 0; position < this.fields.length; position++) {
+      const value = row[this.fields[position]];
+      if (value === null || typeof value === 'undefined') {
+        return undefined;
+      }
+      const key = (0, _compareAs.compareAs)(this.types[position], value);
+      if (typeof key === 'object') {
+        return undefined;
+      }
+      typed.push([typeof key, key]);
+    }
+    return JSON.stringify(typed);
+  }
+  add(row) {
+    const key = this.keyOf(row);
+    if (typeof key !== 'undefined') {
+      this.seen.add(key);
+    }
+  }
+  has(row) {
+    const key = this.keyOf(row);
+    return typeof key !== 'undefined' && this.seen.has(key);
+  }
+}
+/**
+ * One insert statement's view of the keys it checks, so that each index is read and parsed once for all its rows
+ * instead of once per row, and the record file and its indexes are written once at the end instead of once per row.
+ *
+ * It answers every check exactly as reading the files after each row would (as inserts did one row at a time): the
+ * record's own indexes are held in memory (see IndexBuffer) and gain each row as it's added, so a row is checked
+ * against the rows before it in the statement too; another record's index is read once, since an insert never changes
+ * another record. Where a check can't use an index it looks at every row in storage as before, plus this statement's
+ * rows, which aren't in storage until the end.
+ */
+class InsertStatement {
+  constructor(entity, record) {
+    this.rows = [];
+    this.buffers = new Map();
+    this.unreadable = new Set();
+    this.otherLookups = new Map();
+    this.scans = new Map();
+    this.entity = entity;
+    this.record = record;
+    this.hadEntries = (record.entries || []).length > 0;
+  }
+  /**
+   * Read the record's index files, once. An index that can't be read starts empty (as updateIndex starts one) and its
+   * key is checked by looking at every row.
+   */
+  async load() {
+    for (const key of this.record.keys) {
+      if (!key.lookup) {
+        continue;
+      }
+      let entries = [];
+      try {
+        entries = await (0, _storage.readJson)(`__indexes/${this.record.path}/${key.lookup}`);
+      } catch (error) {
+        this.unreadable.add(key);
+      }
+      this.buffers.set(key, new _indexBuffer.IndexBuffer(Array.isArray(entries) ? entries : [], (0, _fieldType.keyValueType)(this.record, key)));
+    }
+  }
+  /**
+   * How to look up a non-null value of `property` in the record being inserted into: through its index buffer, which
+   * holds the statement's rows so far. indexHas reads a record with no entries as holding nothing, whatever its index
+   * file says, so until this statement adds a row to a record that had none, nothing is found.
+   * @param property
+   */
+  ownLookup(property) {
+    const key = this.record.keys.find(key => (0, _isSingleFieldKey.isSingleFieldKey)(key, property) && key.lookup);
+    if (!key || this.unreadable.has(key)) {
+      return () => undefined;
+    }
+    const buffer = this.buffers.get(key);
+    return value => (this.hadEntries || this.rows.length > 0) && buffer.has(value);
+  }
+  /**
+   * How to look up a non-null value of `property` in another record, read once (an insert doesn't change it).
+   * @param entity
+   * @param property
+   */
+  otherLookup(entity, property) {
+    const name = `${entity}\u0000${property}`;
+    if (!this.otherLookups.has(name)) {
+      this.otherLookups.set(name, (async () => {
+        const record = await (0, _retrieveRecord.retrieveRecord)(entity);
+        const key = record.keys.find(key => (0, _isSingleFieldKey.isSingleFieldKey)(key, property) && key.lookup);
+        if (!key) {
+          return () => undefined;
+        }
+        if (!(record.entries || []).length) {
+          return () => false;
+        }
+        let entries;
+        try {
+          entries = await (0, _storage.readJson)(`__indexes/${record.path}/${key.lookup}`);
+        } catch (error) {
+          return () => undefined;
+        }
+        const buffer = new _indexBuffer.IndexBuffer(Array.isArray(entries) ? entries : [], (0, _fieldType.fieldType)(record, property));
+        return value => buffer.has(value);
+      })());
+    }
+    return this.otherLookups.get(name);
+  }
+  /**
+   * The values of these fields in every row of `entity` (read once per statement), plus, for the record being inserted
+   * into, every row this statement adds; undefined when a field's type can't be compared by key.
+   * @param entity
+   * @param fields
+   */
+  scan(entity, fields) {
+    const name = `${entity}\u0000${fields.join('\u0000')}`;
+    if (!this.scans.has(name)) {
+      this.scans.set(name, (async () => {
+        const record = entity === this.entity ? this.record : await (0, _retrieveRecord.retrieveRecord)(entity);
+        if (!RowValues.usable(record, fields)) {
+          return undefined;
+        }
+        const values = new RowValues(record, fields);
+        // The record's rows in storage: this statement's own rows aren't there until the end, and are added as they come
+        for (const row of await (0, _retrieveRecords.retrieveRecords)(entity)) {
+          values.add(row);
+        }
+        if (entity === this.entity) {
+          this.rows.forEach(row => values.add(row));
+        }
+        return values;
+      })());
+    }
+    return this.scans.get(name);
+  }
+  /**
+   * Whether one of this statement's rows already holds these values in every one of these fields (none of them null).
+   * @param fields
+   * @param values
+   */
+  claimed(fields, values) {
+    return this.rows.some(row => fields.every(field => row[field] !== null && typeof row[field] !== 'undefined' && (0, _compareAs.sameValue)((0, _fieldType.fieldType)(this.record, field), row[field], values[field])));
+  }
+  /**
+   * Whether no row (in storage or earlier in this statement) holds these values for these fields, as keyUnique answers
+   * it after each earlier row is written.
+   * @param fields
+   * @param values
+   */
+  async isUnique(fields, values) {
+    if (fields.length === 1) {
+      const found = this.ownLookup(fields[0])(values[fields[0]]);
+      if (typeof found !== 'undefined') {
+        return !found;
+      }
+    }
+    const scanned = await this.scan(this.entity, fields);
+    if (scanned) {
+      return !scanned.has(values);
+    }
+    return (await (0, _keyUnique.keyUnique)(this.entity, fields, fields.map(field => values[field]))) && !this.claimed(fields, values);
+  }
+  /**
+   * Whether `entity` holds `value` in `property`, as foreignKeyExists answers it after each earlier row is written: a
+   * row may reference one added earlier in the same statement when the key points at its own record.
+   * @param entity
+   * @param property
+   * @param value
+   */
+  async exists(entity, property, value) {
+    const own = entity === this.entity;
+    const found = own ? this.ownLookup(property)(value) : (await this.otherLookup(entity, property))(value);
+    if (typeof found !== 'undefined') {
+      return found;
+    }
+    const scanned = await this.scan(entity, [property]);
+    if (scanned) {
+      return scanned.has({
+        [property]: value
+      });
+    }
+    return (await (0, _foreignKeyExists.foreignKeyExists)(entity, property, value)) || own && this.claimed([property], {
+      [property]: value
+    });
+  }
+  /**
+   * Note a row added to the statement, filed under `fileName`, in the record's entries and every index buffer.
+   * @param entityValues
+   * @param fileName
+   */
+  async add(entityValues, fileName) {
+    this.rows.push(entityValues);
+    // Scans already built for this record hold the rows in storage; a row added now joins them
+    for (const [name, scanned] of this.scans) {
+      if (name.startsWith(`${this.entity}\u0000`)) {
+        (await scanned)?.add(entityValues);
+      }
+    }
+    this.record.entries.push(fileName);
+    for (const [key, buffer] of this.buffers) {
+      buffer.add((0, _computeKeyValue.computeKeyValue)(key.fields, entityValues), fileName);
+    }
+  }
+  /**
+   * Write the record file and each index once, with every row the statement added.
+   */
+  async write() {
+    await (0, _storage.writeJson)(`${this.record.path}.json`, this.record);
+    if (this.buffers.size) {
+      await (0, _storage.makeDirectory)(`__indexes/${this.record.path}`);
+    }
+    for (const [key, buffer] of this.buffers) {
+      await (0, _storage.writeJson)(`__indexes/${this.record.path}/${key.lookup}`, buffer.entries);
+    }
+  }
+}
+/**
+ * A new id for an autoGenerate field that no row holds yet, in storage or earlier in the statement.
+ * @param statement
+ * @param field
+ */
+const generateKey = async (statement, field) => {
+  let uuid;
+  do {
+    uuid = crypto.randomUUID();
+  } while (!(await statement.isUnique([field], {
+    [field]: uuid
+  })));
+  return uuid;
+};
 /**
  * Build one entity's stored values from the raw input values, generating any autoGenerate
- * fields and applying field defaults, then throw if a required field still has no value.
- * @param entity
- * @param record
+ * fields and applying field defaults, then throw if a required field still has no value. A null
+ * in an optional field that isn't nullable means the same as leaving it out (the field can't
+ * hold null, so nothing is lost): it isn't stored, and a default fills it in as for a field left out.
+ * @param statement
  * @param rowValues
  */
-const buildEntityValues = async (entity, record, rowValues = {}) => {
+const buildEntityValues = async (statement, rowValues = {}) => {
   const entityValues = {};
-  for (const field of record.definition) {
+  for (const field of statement.record.definition) {
     let value = rowValues[field.name];
+    if (value === null && field.optional && !field.nullable) {
+      value = undefined;
+    }
     if (field.autoGenerate) {
-      value = await (0, _keyGenerate.keyGenerate)(entity, [field.name]);
+      value = await generateKey(statement, field.name);
     } else if (Object.prototype.hasOwnProperty.call(field, 'default')) {
       value = (0, _useDefault.useDefault)(field['default'], value);
     }
     if (typeof value === 'undefined') {
       if (!field.optional) {
-        throw new Error(`Missing required field "${field.name}" for ${entity}`);
+        throw new Error(`Missing required field "${field.name}" for ${statement.entity}`);
       }
       continue;
     }
@@ -54,34 +344,30 @@ const buildEntityValues = async (entity, record, rowValues = {}) => {
  * Reject the insert if it collides with an existing primary or unique key value - a single field,
  * or a composite of several for a multi-field key. As in SQL, a value with a null in it is never a
  * duplicate, so a nullable unique field can hold null in any number of rows.
- * @param entity
- * @param record
+ * @param statement
  * @param entityValues
  */
-const assertUnique = async (entity, record, entityValues) => {
-  for (const key of record.keys) {
+const assertUnique = async (statement, entityValues) => {
+  for (const key of statement.record.keys) {
     if (key.type !== 'primary' && key.type !== 'unique') {
       continue;
     }
-    const checkKeys = key.fields.map(field => entityValues[field]);
-    if (checkKeys.some(value => value === null)) {
+    if (key.fields.some(field => entityValues[field] === null)) {
       continue;
     }
-    const isUnique = await (0, _keyUnique.keyUnique)(entity, key.fields, checkKeys);
-    if (!isUnique) {
-      throw new Error(`Duplicate value for ${key.type} key "${key.fields.join(', ')}" on ${entity}`);
+    if (!(await statement.isUnique(key.fields, entityValues))) {
+      throw new Error(`Duplicate value for ${key.type} key "${key.fields.join(', ')}" on ${statement.entity}`);
     }
   }
 };
 /**
  * Reject the insert if a single-field foreign key's value doesn't exist in the record it
  * references. A null references nothing, so it isn't checked (as in SQL).
- * @param entity
- * @param record
+ * @param statement
  * @param entityValues
  */
-const assertForeignKeys = async (entity, record, entityValues) => {
-  for (const key of record.keys) {
+const assertForeignKeys = async (statement, entityValues) => {
+  for (const key of statement.record.keys) {
     if (key.type !== 'foreign' || key.fields.length !== 1 || !key.references || key.references.length !== 1) {
       continue;
     }
@@ -93,14 +379,20 @@ const assertForeignKeys = async (entity, record, entityValues) => {
       entity: refEntity,
       property: refProperty
     } = (0, _splitEntityProperty.splitEntityProperty)(key.references[0].fields[0]);
-    const exists = await (0, _foreignKeyExists.foreignKeyExists)(refEntity, refProperty, entityValues[field]);
-    if (!exists) {
-      throw new Error(`Foreign key "${field}" on ${entity} references a nonexistent ${refEntity}.${refProperty} = ${entityValues[field]}`);
+    if (!(await statement.exists(refEntity, refProperty, entityValues[field]))) {
+      throw new Error(`Foreign key "${field}" on ${statement.entity} references a nonexistent ${refEntity}.${refProperty} = ${entityValues[field]}`);
     }
   }
 };
+/**
+ * Insert rows into a record, all in one statement: every row is checked (against the rows in storage and the rows
+ * before it in the statement) and its entry file written, then the record file and each index are written once.
+ * Inside a write set (as every query runs, see withWriteSet) a row that fails leaves none of them written.
+ * @param entity
+ * @param values - The rows, as field -> value.
+ * @param dataSet
+ */
 const insertEntity = async (entity = '', values = [], dataSet = {}) => {
-  const databasePath = (0, _config.getSetting)('databasePath', 'database/');
   const record = await (0, _retrieveRecord.retrieveRecord)(entity);
   const primaryKey = record.keys.find(key => key.type === 'primary' && key.fields.length === 1);
   if (!Object.prototype.hasOwnProperty.call(dataSet, entity)) {
@@ -108,22 +400,27 @@ const insertEntity = async (entity = '', values = [], dataSet = {}) => {
   }
   // The record's folder can be missing even though the record exists: git doesn't keep empty folders, so a database
   // committed before its first insert comes back from a fresh clone without it
-  await mkdir(`${databasePath}${record.path}`, {
-    recursive: true
-  });
+  await (0, _storage.makeDirectory)(record.path);
+  if (!values.length) {
+    return dataSet;
+  }
+  // A field the record doesn't define would otherwise be dropped without a word
   for (const rowValues of values) {
-    const entityValues = await buildEntityValues(entity, record, rowValues);
-    await assertUnique(entity, record, entityValues);
-    await assertForeignKeys(entity, record, entityValues);
+    (0, _knownFields.assertFieldsDefined)(entity, record, Object.keys(rowValues || {}), 'insert');
+  }
+  await (0, _packedTable.discardPacked)(record.path);
+  const statement = new InsertStatement(entity, record);
+  await statement.load();
+  for (const rowValues of values) {
+    const entityValues = await buildEntityValues(statement, rowValues);
+    await assertUnique(statement, entityValues);
+    await assertForeignKeys(statement, entityValues);
     const fileName = `${primaryKey ? entityValues[primaryKey.fields[0]] : crypto.randomUUID()}.json`;
-    await writeFile(`${databasePath}${record.path}/${fileName}`, JSON.stringify(entityValues, null, 2));
-    record.entries.push(fileName);
-    await writeFile(`${databasePath}${record.path}.json`, JSON.stringify(record, null, 2));
-    for (const key of record.keys) {
-      await (0, _updateIndex.updateIndex)(record.path, key, (0, _computeKeyValue.computeKeyValue)(key.fields, entityValues), fileName, (0, _fieldType.keyValueType)(record, key));
-    }
+    await (0, _storage.writeJson)(`${record.path}/${fileName}`, entityValues);
+    await statement.add(entityValues, fileName);
     dataSet[entity].push(entityValues);
   }
+  await statement.write();
   return dataSet;
 };
 exports.insertEntity = insertEntity;

package/dist/data/queries/insertEntity.min.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.insertEntity=void 0,require("core-js/modules/es.json.stringify.js"),require("core-js/modules/esnext.iterator.constructor.js"),require("core-js/modules/esnext.iterator.find.js"),require("core-js/modules/esnext.iterator.map.js"),require("core-js/modules/esnext.iterator.some.js");var _retrieveRecord=require("../utilities/retrieveRecord"),_updateIndex=require("../utilities/updateIndex"),_keyGenerate=require("../introspect/policies/keyGenerate"),_keyUnique=require("../introspect/policies/keyUnique"),_useDefault=require("../introspect/policies/useDefault"),_typeCheck=require("../introspect/policies/typeCheck"),_foreignKeyExists=require("../introspect/policies/foreignKeyExists"),_splitEntityProperty=require("../utilities/parsers/splitEntityProperty"),_computeKeyValue=require("../utilities/computeKeyValue"),_fieldType=require("../utilities/fieldType"),_config=require("../utilities/config");const{writeFile:writeFile,mkdir:mkdir}=require("fs/promises"),buildEntityValues=async(e,t,i={})=>{const r={};for(const s of t.definition){let t=i[s.name];if(s.autoGenerate?t=await(0,_keyGenerate.keyGenerate)(e,[s.name]):Object.prototype.hasOwnProperty.call(s,"default")&&(t=(0,_useDefault.useDefault)(s.default,t)),void 0!==t)r[s.name]=(0,_typeCheck.typeCheck)(s.type,t,s.nullable);else if(!s.optional)throw new Error(`Missing required field "${s.name}" for ${e}`)}return r},assertUnique=async(e,t,i)=>{for(const r of t.keys){if("primary"!==r.type&&"unique"!==r.type)continue;const t=r.fields.map((e=>i[e]));if(t.some((e=>null===e)))continue;if(!await(0,_keyUnique.keyUnique)(e,r.fields,t))throw new Error(`Duplicate value for ${r.type} key "${r.fields.join(", ")}" on ${e}`)}},assertForeignKeys=async(e,t,i)=>{for(const r of t.keys){if("foreign"!==r.type||1!==r.fields.length||!r.references||1!==r.references.length)continue;const t=r.fields[0];if(!Object.prototype.hasOwnProperty.call(i,t)||null===i[t])continue;const{entity:s,property:n}=(0,_splitEntityProperty.splitEntityProperty)(r.references[0].fields[0]);if(!await(0,_foreignKeyExists.foreignKeyExists)(s,n,i[t]))throw new Error(`Foreign key "${t}" on ${e} references a nonexistent ${s}.${n} = ${i[t]}`)}},insertEntity=async(e="",t=[],i={})=>{const r=(0,_config.getSetting)("databasePath","database/"),s=await(0,_retrieveRecord.retrieveRecord)(e),n=s.keys.find((e=>"primary"===e.type&&1===e.fields.length));Object.prototype.hasOwnProperty.call(i,e)||(i[e]=[]),await mkdir(`${r}${s.path}`,{recursive:!0});for(const o of t){const t=await buildEntityValues(e,s,o);await assertUnique(e,s,t),await assertForeignKeys(e,s,t);const a=`${n?t[n.fields[0]]:crypto.randomUUID()}.json`;await writeFile(`${r}${s.path}/${a}`,JSON.stringify(t,null,2)),s.entries.push(a),await writeFile(`${r}${s.path}.json`,JSON.stringify(s,null,2));for(const e of s.keys)await(0,_updateIndex.updateIndex)(s.path,e,(0,_computeKeyValue.computeKeyValue)(e.fields,t),a,(0,_fieldType.keyValueType)(s,e));i[e].push(t)}return i};exports.insertEntity=insertEntity;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.insertEntity=void 0,require("core-js/modules/es.array.includes.js"),require("core-js/modules/es.json.stringify.js"),require("core-js/modules/esnext.iterator.constructor.js"),require("core-js/modules/esnext.iterator.every.js"),require("core-js/modules/esnext.iterator.find.js"),require("core-js/modules/esnext.iterator.for-each.js"),require("core-js/modules/esnext.iterator.map.js"),require("core-js/modules/esnext.iterator.some.js"),require("core-js/modules/esnext.map.delete-all.js"),require("core-js/modules/esnext.map.every.js"),require("core-js/modules/esnext.map.filter.js"),require("core-js/modules/esnext.map.find.js"),require("core-js/modules/esnext.map.find-key.js"),require("core-js/modules/esnext.map.includes.js"),require("core-js/modules/esnext.map.key-of.js"),require("core-js/modules/esnext.map.map-keys.js"),require("core-js/modules/esnext.map.map-values.js"),require("core-js/modules/esnext.map.merge.js"),require("core-js/modules/esnext.map.reduce.js"),require("core-js/modules/esnext.map.some.js"),require("core-js/modules/esnext.map.update.js"),require("core-js/modules/esnext.set.add-all.js"),require("core-js/modules/esnext.set.delete-all.js"),require("core-js/modules/esnext.set.difference.js"),require("core-js/modules/esnext.set.every.js"),require("core-js/modules/esnext.set.filter.js"),require("core-js/modules/esnext.set.find.js"),require("core-js/modules/esnext.set.intersection.js"),require("core-js/modules/esnext.set.is-disjoint-from.js"),require("core-js/modules/esnext.set.is-subset-of.js"),require("core-js/modules/esnext.set.is-superset-of.js"),require("core-js/modules/esnext.set.join.js"),require("core-js/modules/esnext.set.map.js"),require("core-js/modules/esnext.set.reduce.js"),require("core-js/modules/esnext.set.some.js"),require("core-js/modules/esnext.set.symmetric-difference.js"),require("core-js/modules/esnext.set.union.js");var _retrieveRecord=require("../utilities/retrieveRecord"),_keyUnique=require("../introspect/policies/keyUnique"),_useDefault=require("../introspect/policies/useDefault"),_typeCheck=require("../introspect/policies/typeCheck"),_foreignKeyExists=require("../introspect/policies/foreignKeyExists"),_splitEntityProperty=require("../utilities/parsers/splitEntityProperty"),_computeKeyValue=require("../utilities/computeKeyValue"),_fieldType=require("../utilities/fieldType"),_compareAs=require("../utilities/compareAs"),_retrieveRecords=require("../utilities/retrieveRecords"),_isSingleFieldKey=require("../utilities/isSingleFieldKey"),_indexBuffer=require("../utilities/indexBuffer"),_storage=require("../storage"),_packedTable=require("../utilities/packedTable"),_knownFields=require("../utilities/knownFields");class RowValues{constructor(e,s){this.seen=new Set,this.fields=s,this.types=s.map((s=>(0,_fieldType.fieldType)(e,s)))}static usable(e,s){return s.every((s=>_compareAs.comparableTypes.includes((0,_fieldType.fieldType)(e,s))))}keyOf(e){const s=[];for(let t=0;t<this.fields.length;t++){const r=e[this.fields[t]];if(null==r)return;const i=(0,_compareAs.compareAs)(this.types[t],r);if("object"==typeof i)return;s.push([typeof i,i])}return JSON.stringify(s)}add(e){const s=this.keyOf(e);void 0!==s&&this.seen.add(s)}has(e){const s=this.keyOf(e);return void 0!==s&&this.seen.has(s)}}class InsertStatement{constructor(e,s){this.rows=[],this.buffers=new Map,this.unreadable=new Set,this.otherLookups=new Map,this.scans=new Map,this.entity=e,this.record=s,this.hadEntries=(s.entries||[]).length>0}async load(){for(const e of this.record.keys){if(!e.lookup)continue;let s=[];try{s=await(0,_storage.readJson)(`__indexes/${this.record.path}/${e.lookup}`)}catch(s){this.unreadable.add(e)}this.buffers.set(e,new _indexBuffer.IndexBuffer(Array.isArray(s)?s:[],(0,_fieldType.keyValueType)(this.record,e)))}}ownLookup(e){const s=this.record.keys.find((s=>(0,_isSingleFieldKey.isSingleFieldKey)(s,e)&&s.lookup));if(!s||this.unreadable.has(s))return()=>{};const t=this.buffers.get(s);return e=>(this.hadEntries||this.rows.length>0)&&t.has(e)}otherLookup(e,s){const t=`${e}\0${s}`;return this.otherLookups.has(t)||this.otherLookups.set(t,(async()=>{const t=await(0,_retrieveRecord.retrieveRecord)(e),r=t.keys.find((e=>(0,_isSingleFieldKey.isSingleFieldKey)(e,s)&&e.lookup));if(!r)return()=>{};if(!(t.entries||[]).length)return()=>!1;let i;try{i=await(0,_storage.readJson)(`__indexes/${t.path}/${r.lookup}`)}catch(e){return()=>{}}const o=new _indexBuffer.IndexBuffer(Array.isArray(i)?i:[],(0,_fieldType.fieldType)(t,s));return e=>o.has(e)})()),this.otherLookups.get(t)}scan(e,s){const t=`${e}\0${s.join("\0")}`;return this.scans.has(t)||this.scans.set(t,(async()=>{const t=e===this.entity?this.record:await(0,_retrieveRecord.retrieveRecord)(e);if(!RowValues.usable(t,s))return;const r=new RowValues(t,s);for(const s of await(0,_retrieveRecords.retrieveRecords)(e))r.add(s);return e===this.entity&&this.rows.forEach((e=>r.add(e))),r})()),this.scans.get(t)}claimed(e,s){return this.rows.some((t=>e.every((e=>null!==t[e]&&void 0!==t[e]&&(0,_compareAs.sameValue)((0,_fieldType.fieldType)(this.record,e),t[e],s[e])))))}async isUnique(e,s){if(1===e.length){const t=this.ownLookup(e[0])(s[e[0]]);if(void 0!==t)return!t}const t=await this.scan(this.entity,e);return t?!t.has(s):await(0,_keyUnique.keyUnique)(this.entity,e,e.map((e=>s[e])))&&!this.claimed(e,s)}async exists(e,s,t){const r=e===this.entity,i=r?this.ownLookup(s)(t):(await this.otherLookup(e,s))(t);if(void 0!==i)return i;const o=await this.scan(e,[s]);return o?o.has({[s]:t}):await(0,_foreignKeyExists.foreignKeyExists)(e,s,t)||r&&this.claimed([s],{[s]:t})}async add(e,s){this.rows.push(e);for(const[s,t]of this.scans)s.startsWith(`${this.entity}\0`)&&(await t)?.add(e);this.record.entries.push(s);for(const[t,r]of this.buffers)r.add((0,_computeKeyValue.computeKeyValue)(t.fields,e),s)}async write(){await(0,_storage.writeJson)(`${this.record.path}.json`,this.record),this.buffers.size&&await(0,_storage.makeDirectory)(`__indexes/${this.record.path}`);for(const[e,s]of this.buffers)await(0,_storage.writeJson)(`__indexes/${this.record.path}/${e.lookup}`,s.entries)}}const generateKey=async(e,s)=>{let t;do{t=crypto.randomUUID()}while(!await e.isUnique([s],{[s]:t}));return t},buildEntityValues=async(e,s={})=>{const t={};for(const r of e.record.definition){let i=s[r.name];if(null===i&&r.optional&&!r.nullable&&(i=void 0),r.autoGenerate?i=await generateKey(e,r.name):Object.prototype.hasOwnProperty.call(r,"default")&&(i=(0,_useDefault.useDefault)(r.default,i)),void 0!==i)t[r.name]=(0,_typeCheck.typeCheck)(r.type,i,r.nullable);else if(!r.optional)throw new Error(`Missing required field "${r.name}" for ${e.entity}`)}return t},assertUnique=async(e,s)=>{for(const t of e.record.keys)if(!("primary"!==t.type&&"unique"!==t.type||t.fields.some((e=>null===s[e]))||await e.isUnique(t.fields,s)))throw new Error(`Duplicate value for ${t.type} key "${t.fields.join(", ")}" on ${e.entity}`)},assertForeignKeys=async(e,s)=>{for(const t of e.record.keys){if("foreign"!==t.type||1!==t.fields.length||!t.references||1!==t.references.length)continue;const r=t.fields[0];if(!Object.prototype.hasOwnProperty.call(s,r)||null===s[r])continue;const{entity:i,property:o}=(0,_splitEntityProperty.splitEntityProperty)(t.references[0].fields[0]);if(!await e.exists(i,o,s[r]))throw new Error(`Foreign key "${r}" on ${e.entity} references a nonexistent ${i}.${o} = ${s[r]}`)}},insertEntity=async(e="",s=[],t={})=>{const r=await(0,_retrieveRecord.retrieveRecord)(e),i=r.keys.find((e=>"primary"===e.type&&1===e.fields.length));if(Object.prototype.hasOwnProperty.call(t,e)||(t[e]=[]),await(0,_storage.makeDirectory)(r.path),!s.length)return t;for(const t of s)(0,_knownFields.assertFieldsDefined)(e,r,Object.keys(t||{}),"insert");await(0,_packedTable.discardPacked)(r.path);const o=new InsertStatement(e,r);await o.load();for(const n of s){const s=await buildEntityValues(o,n);await assertUnique(o,s),await assertForeignKeys(o,s);const a=`${i?s[i.fields[0]]:crypto.randomUUID()}.json`;await(0,_storage.writeJson)(`${r.path}/${a}`,s),await o.add(s,a),t[e].push(s)}return await o.write(),t};exports.insertEntity=insertEntity;

package/dist/data/queries/joinEntity.d.ts

@@ -1,2 +1,17 @@
+/**
+ * Attach to each row of `entity` the rows of `joinEntity` its join reaches, through the `on` clauses in order.
+ *
+ * Each clause narrows both its records to the rows that match the other side: first the rows of the left one that
+ * match a right one, then the right ones that match those (see semiJoin for the order). A row's join starts from the
+ * row alone and from the rows read for every other record (see joinBases), so each record is read once for the whole
+ * join and each lookup is built once, rather than every loaded row being copied for each row joined.
+ * @param entity
+ * @param joinEntity
+ * @param dataSets
+ * @param joinClauses
+ * @param entityAlreadyResolved - Whether the where clause already narrowed entity's rows (then they aren't read again:
+ * an empty list reads as "not loaded yet", see readEntity).
+ * @param joinAlreadyResolved - The same for joinEntity.
+ */
 export declare const joinEntity: (entity: any, joinEntity: any, dataSets?: {}, joinClauses?: any[], entityAlreadyResolved?: boolean, joinAlreadyResolved?: boolean) => Promise<{}>;
 //# sourceMappingURL=joinEntity.d.ts.map

package/dist/data/queries/joinEntity.js

@@ -4,31 +4,107 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.joinEntity = void 0;
-var _siFunciona = _interopRequireDefault(require("si-funciona"));
+require("core-js/modules/esnext.iterator.constructor.js");
+require("core-js/modules/esnext.iterator.map.js");
+require("core-js/modules/esnext.map.delete-all.js");
+require("core-js/modules/esnext.map.every.js");
+require("core-js/modules/esnext.map.filter.js");
+require("core-js/modules/esnext.map.find.js");
+require("core-js/modules/esnext.map.find-key.js");
+require("core-js/modules/esnext.map.includes.js");
+require("core-js/modules/esnext.map.key-of.js");
+require("core-js/modules/esnext.map.map-keys.js");
+require("core-js/modules/esnext.map.map-values.js");
+require("core-js/modules/esnext.map.merge.js");
+require("core-js/modules/esnext.map.reduce.js");
+require("core-js/modules/esnext.map.some.js");
+require("core-js/modules/esnext.map.update.js");
 var _readEntity = require("./readEntity");
-var _useJoinClause = require("../utilities/useJoinClause");
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+var _retrieveRecord = require("../utilities/retrieveRecord");
+var _splitEntityProperty = require("../utilities/parsers/splitEntityProperty");
+var _fieldType = require("../utilities/fieldType");
+var _semiJoin = require("../utilities/semiJoin");
+var _joinBases = require("../utilities/joinBases");
+/**
+ * Each `on` clause with the types its two sides are compared by. Each record is read here, before its type is looked
+ * up, so a record that doesn't exist fails with "Could not read <record>.json", as it always has.
+ * @param joinClauses
+ */
+const joinSteps = async joinClauses => {
+  const steps = [];
+  for (const {
+    propertyA,
+    comparator,
+    propertyB
+  } of joinClauses) {
+    const {
+      entity: entityA,
+      property: a
+    } = (0, _splitEntityProperty.splitEntityProperty)(propertyA);
+    const {
+      entity: entityB,
+      property: b
+    } = (0, _splitEntityProperty.splitEntityProperty)(propertyB);
+    steps.push({
+      entityA,
+      propertyA: a,
+      typeA: (0, _fieldType.fieldType)(await (0, _retrieveRecord.retrieveRecord)(entityA), a),
+      comparator,
+      entityB,
+      propertyB: b,
+      typeB: (0, _fieldType.fieldType)(await (0, _retrieveRecord.retrieveRecord)(entityB), b)
+    });
+  }
+  return steps;
+};
+/**
+ * What a row gets for its joined rows. Today every row is kept with the list of rows its join found (none is an empty
+ * list), and a condition on the joined record's fields then drops rows with none (whereTree): an inner join. This is
+ * the one place a join type (left, right, outer) would decide differently.
+ * @param matches - The joined rows the row's join found, shared with other rows' joins until copied here.
+ */
+const joinedRows = matches => matches.map(match => structuredClone(match));
+/**
+ * Attach to each row of `entity` the rows of `joinEntity` its join reaches, through the `on` clauses in order.
+ *
+ * Each clause narrows both its records to the rows that match the other side: first the rows of the left one that
+ * match a right one, then the right ones that match those (see semiJoin for the order). A row's join starts from the
+ * row alone and from the rows read for every other record (see joinBases), so each record is read once for the whole
+ * join and each lookup is built once, rather than every loaded row being copied for each row joined.
+ * @param entity
+ * @param joinEntity
+ * @param dataSets
+ * @param joinClauses
+ * @param entityAlreadyResolved - Whether the where clause already narrowed entity's rows (then they aren't read again:
+ * an empty list reads as "not loaded yet", see readEntity).
+ * @param joinAlreadyResolved - The same for joinEntity.
+ */
 const joinEntity = async (entity, joinEntity, dataSets = {}, joinClauses = [], entityAlreadyResolved = false, joinAlreadyResolved = false) => {
-  // Same ambiguity readEntity() always has (can't tell "never loaded" from "loaded, zero rows
-  // matched") - runQuery already resolved entity's rows via an indexed condition in that second
-  // case, so entityAlreadyResolved tells this call not to reload and silently discard that result.
   if (!entityAlreadyResolved) {
     dataSets = await (0, _readEntity.readEntity)(entity, dataSets);
   }
   if (!joinClauses.length) {
     return dataSets;
   }
-  // The same for joinEntity: an indexed condition on its field may have resolved its rows to none
-  if (!joinAlreadyResolved) {
-    dataSets = await (0, _readEntity.readEntity)(joinEntity, dataSets);
+  const steps = await joinSteps(joinClauses);
+  if (!dataSets[entity].length) {
+    return dataSets;
   }
-  for (const anEntity in dataSets[entity]) {
-    let filterJoins = _siFunciona.default.cloneObject(dataSets);
-    filterJoins[entity] = [dataSets[entity][anEntity]];
-    for (const joinClause of joinClauses) {
-      filterJoins = await (0, _useJoinClause.useJoinClause)(joinClause, dataSets, filterJoins);
+  const bases = await (0, _joinBases.joinBases)(entity, joinEntity, dataSets, steps, joinAlreadyResolved);
+  for (const row of dataSets[entity]) {
+    const sides = new Map(bases);
+    sides.set(entity, {
+      rows: [row]
+    });
+    for (const step of steps) {
+      sides.set(step.entityA, {
+        rows: (0, _semiJoin.semiJoin)(sides.get(step.entityA), sides.get(step.entityB), step.propertyA, step.propertyB, step.comparator, step.typeA)
+      });
+      sides.set(step.entityB, {
+        rows: (0, _semiJoin.semiJoin)(sides.get(step.entityB), sides.get(step.entityA), step.propertyB, step.propertyA, step.comparator, step.typeB)
+      });
     }
-    dataSets[entity][anEntity][joinEntity] = filterJoins[joinEntity];
+    row[joinEntity] = joinedRows(sides.get(joinEntity).rows);
   }
   return dataSets;
 };

package/dist/data/queries/joinEntity.min.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.joinEntity=void 0;var _siFunciona=_interopRequireDefault(require("si-funciona")),_readEntity=require("./readEntity"),_useJoinClause=require("../utilities/useJoinClause");function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}const joinEntity=async(e,t,i={},n=[],r=!1,u=!1)=>{if(r||(i=await(0,_readEntity.readEntity)(e,i)),!n.length)return i;u||(i=await(0,_readEntity.readEntity)(t,i));for(const r in i[e]){let u=_siFunciona.default.cloneObject(i);u[e]=[i[e][r]];for(const e of n)u=await(0,_useJoinClause.useJoinClause)(e,i,u);i[e][r][t]=u[t]}return i};exports.joinEntity=joinEntity;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.joinEntity=void 0,require("core-js/modules/esnext.iterator.constructor.js"),require("core-js/modules/esnext.iterator.map.js"),require("core-js/modules/esnext.map.delete-all.js"),require("core-js/modules/esnext.map.every.js"),require("core-js/modules/esnext.map.filter.js"),require("core-js/modules/esnext.map.find.js"),require("core-js/modules/esnext.map.find-key.js"),require("core-js/modules/esnext.map.includes.js"),require("core-js/modules/esnext.map.key-of.js"),require("core-js/modules/esnext.map.map-keys.js"),require("core-js/modules/esnext.map.map-values.js"),require("core-js/modules/esnext.map.merge.js"),require("core-js/modules/esnext.map.reduce.js"),require("core-js/modules/esnext.map.some.js"),require("core-js/modules/esnext.map.update.js");var _readEntity=require("./readEntity"),_retrieveRecord=require("../utilities/retrieveRecord"),_splitEntityProperty=require("../utilities/parsers/splitEntityProperty"),_fieldType=require("../utilities/fieldType"),_semiJoin=require("../utilities/semiJoin"),_joinBases=require("../utilities/joinBases");const joinSteps=async e=>{const t=[];for(const{propertyA:r,comparator:s,propertyB:i}of e){const{entity:e,property:o}=(0,_splitEntityProperty.splitEntityProperty)(r),{entity:n,property:p}=(0,_splitEntityProperty.splitEntityProperty)(i);t.push({entityA:e,propertyA:o,typeA:(0,_fieldType.fieldType)(await(0,_retrieveRecord.retrieveRecord)(e),o),comparator:s,entityB:n,propertyB:p,typeB:(0,_fieldType.fieldType)(await(0,_retrieveRecord.retrieveRecord)(n),p)})}return t},joinedRows=e=>e.map((e=>structuredClone(e))),joinEntity=async(e,t,r={},s=[],i=!1,o=!1)=>{if(i||(r=await(0,_readEntity.readEntity)(e,r)),!s.length)return r;const n=await joinSteps(s);if(!r[e].length)return r;const p=await(0,_joinBases.joinBases)(e,t,r,n,o);for(const s of r[e]){const r=new Map(p);r.set(e,{rows:[s]});for(const e of n)r.set(e.entityA,{rows:(0,_semiJoin.semiJoin)(r.get(e.entityA),r.get(e.entityB),e.propertyA,e.propertyB,e.comparator,e.typeA)}),r.set(e.entityB,{rows:(0,_semiJoin.semiJoin)(r.get(e.entityB),r.get(e.entityA),e.propertyB,e.propertyA,e.comparator,e.typeB)});s[t]=joinedRows(r.get(t).rows)}return r};exports.joinEntity=joinEntity;

package/dist/data/queries/query.d.ts

@@ -1,7 +1,8 @@
 /**
  * Parse a query string (one query, or several separated by `;`) and run each query in turn, resolving to one result
  * per query. The queries of one call run together, under the database lock: no other call's queries run between
- * them (see withDatabaseLock).
+ * them (see withDatabaseLock). Each query is all or nothing: one that fails changes nothing, and the queries before it
+ * keep their changes (see withWriteSet).
  * @param queryString
  */
 export declare const query: (queryString?: string) => Promise<any[]>;