json-fs-query 3.2.0 → 3.3.0

package/dist/data/queries/insertEntity.js

@@ -4,13 +4,44 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.insertEntity = void 0;
+require("core-js/modules/es.array.includes.js");
+require("core-js/modules/es.json.stringify.js");
 require("core-js/modules/esnext.iterator.constructor.js");
+require("core-js/modules/esnext.iterator.every.js");
 require("core-js/modules/esnext.iterator.find.js");
+require("core-js/modules/esnext.iterator.for-each.js");
 require("core-js/modules/esnext.iterator.map.js");
 require("core-js/modules/esnext.iterator.some.js");
+require("core-js/modules/esnext.map.delete-all.js");
+require("core-js/modules/esnext.map.every.js");
+require("core-js/modules/esnext.map.filter.js");
+require("core-js/modules/esnext.map.find.js");
+require("core-js/modules/esnext.map.find-key.js");
+require("core-js/modules/esnext.map.includes.js");
+require("core-js/modules/esnext.map.key-of.js");
+require("core-js/modules/esnext.map.map-keys.js");
+require("core-js/modules/esnext.map.map-values.js");
+require("core-js/modules/esnext.map.merge.js");
+require("core-js/modules/esnext.map.reduce.js");
+require("core-js/modules/esnext.map.some.js");
+require("core-js/modules/esnext.map.update.js");
+require("core-js/modules/esnext.set.add-all.js");
+require("core-js/modules/esnext.set.delete-all.js");
+require("core-js/modules/esnext.set.difference.js");
+require("core-js/modules/esnext.set.every.js");
+require("core-js/modules/esnext.set.filter.js");
+require("core-js/modules/esnext.set.find.js");
+require("core-js/modules/esnext.set.intersection.js");
+require("core-js/modules/esnext.set.is-disjoint-from.js");
+require("core-js/modules/esnext.set.is-subset-of.js");
+require("core-js/modules/esnext.set.is-superset-of.js");
+require("core-js/modules/esnext.set.join.js");
+require("core-js/modules/esnext.set.map.js");
+require("core-js/modules/esnext.set.reduce.js");
+require("core-js/modules/esnext.set.some.js");
+require("core-js/modules/esnext.set.symmetric-difference.js");
+require("core-js/modules/esnext.set.union.js");
 var _retrieveRecord = require("../utilities/retrieveRecord");
-var _updateIndex = require("../utilities/updateIndex");
-var _keyGenerate = require("../introspect/policies/keyGenerate");
 var _keyUnique = require("../introspect/policies/keyUnique");
 var _useDefault = require("../introspect/policies/useDefault");
 var _typeCheck = require("../introspect/policies/typeCheck");
@@ -18,26 +49,290 @@ var _foreignKeyExists = require("../introspect/policies/foreignKeyExists");
 var _splitEntityProperty = require("../utilities/parsers/splitEntityProperty");
 var _computeKeyValue = require("../utilities/computeKeyValue");
 var _fieldType = require("../utilities/fieldType");
+var _compareAs = require("../utilities/compareAs");
+var _retrieveRecords = require("../utilities/retrieveRecords");
+var _isSingleFieldKey = require("../utilities/isSingleFieldKey");
+var _indexBuffer = require("../utilities/indexBuffer");
 var _storage = require("../storage");
+var _packedTable = require("../utilities/packedTable");
+var _knownFields = require("../utilities/knownFields");
+/**
+ * The rows of a record that hold each combination of values for some fields, compared field by field by type, as
+ * where's = compares them: built from the rows in storage once per statement, for a key no index can answer (several
+ * fields, or no index file), instead of looking at every row in storage once per inserted row.
+ */
+class RowValues {
+  constructor(record, fields) {
+    this.seen = new Set();
+    this.fields = fields;
+    this.types = fields.map(field => (0, _fieldType.fieldType)(record, field));
+  }
+  /**
+   * Whether these fields' values can be compared this way: every field of a comparable type (see compareAs).
+   * @param record
+   * @param fields
+   */
+  static usable(record, fields) {
+    return fields.every(field => _compareAs.comparableTypes.includes((0, _fieldType.fieldType)(record, field)));
+  }
+  /**
+   * The values' key, or undefined when one is null or missing (where's = never matches those) or compareAs leaves it an
+   * object (compared by identity, so not by a key).
+   * @param row
+   */
+  keyOf(row) {
+    const typed = [];
+    for (let position = 0; position < this.fields.length; position++) {
+      const value = row[this.fields[position]];
+      if (value === null || typeof value === 'undefined') {
+        return undefined;
+      }
+      const key = (0, _compareAs.compareAs)(this.types[position], value);
+      if (typeof key === 'object') {
+        return undefined;
+      }
+      typed.push([typeof key, key]);
+    }
+    return JSON.stringify(typed);
+  }
+  add(row) {
+    const key = this.keyOf(row);
+    if (typeof key !== 'undefined') {
+      this.seen.add(key);
+    }
+  }
+  has(row) {
+    const key = this.keyOf(row);
+    return typeof key !== 'undefined' && this.seen.has(key);
+  }
+}
+/**
+ * One insert statement's view of the keys it checks, so that each index is read and parsed once for all its rows
+ * instead of once per row, and the record file and its indexes are written once at the end instead of once per row.
+ *
+ * It answers every check exactly as reading the files after each row would (as inserts did one row at a time): the
+ * record's own indexes are held in memory (see IndexBuffer) and gain each row as it's added, so a row is checked
+ * against the rows before it in the statement too; another record's index is read once, since an insert never changes
+ * another record. Where a check can't use an index it looks at every row in storage as before, plus this statement's
+ * rows, which aren't in storage until the end.
+ */
+class InsertStatement {
+  constructor(entity, record) {
+    this.rows = [];
+    this.buffers = new Map();
+    this.unreadable = new Set();
+    this.otherLookups = new Map();
+    this.scans = new Map();
+    this.entity = entity;
+    this.record = record;
+    this.hadEntries = (record.entries || []).length > 0;
+  }
+  /**
+   * Read the record's index files, once. An index that can't be read starts empty (as updateIndex starts one) and its
+   * key is checked by looking at every row.
+   */
+  async load() {
+    for (const key of this.record.keys) {
+      if (!key.lookup) {
+        continue;
+      }
+      let entries = [];
+      try {
+        entries = await (0, _storage.readJson)(`__indexes/${this.record.path}/${key.lookup}`);
+      } catch (error) {
+        this.unreadable.add(key);
+      }
+      this.buffers.set(key, new _indexBuffer.IndexBuffer(Array.isArray(entries) ? entries : [], (0, _fieldType.keyValueType)(this.record, key)));
+    }
+  }
+  /**
+   * How to look up a non-null value of `property` in the record being inserted into: through its index buffer, which
+   * holds the statement's rows so far. indexHas reads a record with no entries as holding nothing, whatever its index
+   * file says, so until this statement adds a row to a record that had none, nothing is found.
+   * @param property
+   */
+  ownLookup(property) {
+    const key = this.record.keys.find(key => (0, _isSingleFieldKey.isSingleFieldKey)(key, property) && key.lookup);
+    if (!key || this.unreadable.has(key)) {
+      return () => undefined;
+    }
+    const buffer = this.buffers.get(key);
+    return value => (this.hadEntries || this.rows.length > 0) && buffer.has(value);
+  }
+  /**
+   * How to look up a non-null value of `property` in another record, read once (an insert doesn't change it).
+   * @param entity
+   * @param property
+   */
+  otherLookup(entity, property) {
+    const name = `${entity}\u0000${property}`;
+    if (!this.otherLookups.has(name)) {
+      this.otherLookups.set(name, (async () => {
+        const record = await (0, _retrieveRecord.retrieveRecord)(entity);
+        const key = record.keys.find(key => (0, _isSingleFieldKey.isSingleFieldKey)(key, property) && key.lookup);
+        if (!key) {
+          return () => undefined;
+        }
+        if (!(record.entries || []).length) {
+          return () => false;
+        }
+        let entries;
+        try {
+          entries = await (0, _storage.readJson)(`__indexes/${record.path}/${key.lookup}`);
+        } catch (error) {
+          return () => undefined;
+        }
+        const buffer = new _indexBuffer.IndexBuffer(Array.isArray(entries) ? entries : [], (0, _fieldType.fieldType)(record, property));
+        return value => buffer.has(value);
+      })());
+    }
+    return this.otherLookups.get(name);
+  }
+  /**
+   * The values of these fields in every row of `entity` (read once per statement), plus, for the record being inserted
+   * into, every row this statement adds; undefined when a field's type can't be compared by key.
+   * @param entity
+   * @param fields
+   */
+  scan(entity, fields) {
+    const name = `${entity}\u0000${fields.join('\u0000')}`;
+    if (!this.scans.has(name)) {
+      this.scans.set(name, (async () => {
+        const record = entity === this.entity ? this.record : await (0, _retrieveRecord.retrieveRecord)(entity);
+        if (!RowValues.usable(record, fields)) {
+          return undefined;
+        }
+        const values = new RowValues(record, fields);
+        // The record's rows in storage: this statement's own rows aren't there until the end, and are added as they come
+        for (const row of await (0, _retrieveRecords.retrieveRecords)(entity)) {
+          values.add(row);
+        }
+        if (entity === this.entity) {
+          this.rows.forEach(row => values.add(row));
+        }
+        return values;
+      })());
+    }
+    return this.scans.get(name);
+  }
+  /**
+   * Whether one of this statement's rows already holds these values in every one of these fields (none of them null).
+   * @param fields
+   * @param values
+   */
+  claimed(fields, values) {
+    return this.rows.some(row => fields.every(field => row[field] !== null && typeof row[field] !== 'undefined' && (0, _compareAs.sameValue)((0, _fieldType.fieldType)(this.record, field), row[field], values[field])));
+  }
+  /**
+   * Whether no row (in storage or earlier in this statement) holds these values for these fields, as keyUnique answers
+   * it after each earlier row is written.
+   * @param fields
+   * @param values
+   */
+  async isUnique(fields, values) {
+    if (fields.length === 1) {
+      const found = this.ownLookup(fields[0])(values[fields[0]]);
+      if (typeof found !== 'undefined') {
+        return !found;
+      }
+    }
+    const scanned = await this.scan(this.entity, fields);
+    if (scanned) {
+      return !scanned.has(values);
+    }
+    return (await (0, _keyUnique.keyUnique)(this.entity, fields, fields.map(field => values[field]))) && !this.claimed(fields, values);
+  }
+  /**
+   * Whether `entity` holds `value` in `property`, as foreignKeyExists answers it after each earlier row is written: a
+   * row may reference one added earlier in the same statement when the key points at its own record.
+   * @param entity
+   * @param property
+   * @param value
+   */
+  async exists(entity, property, value) {
+    const own = entity === this.entity;
+    const found = own ? this.ownLookup(property)(value) : (await this.otherLookup(entity, property))(value);
+    if (typeof found !== 'undefined') {
+      return found;
+    }
+    const scanned = await this.scan(entity, [property]);
+    if (scanned) {
+      return scanned.has({
+        [property]: value
+      });
+    }
+    return (await (0, _foreignKeyExists.foreignKeyExists)(entity, property, value)) || own && this.claimed([property], {
+      [property]: value
+    });
+  }
+  /**
+   * Note a row added to the statement, filed under `fileName`, in the record's entries and every index buffer.
+   * @param entityValues
+   * @param fileName
+   */
+  async add(entityValues, fileName) {
+    this.rows.push(entityValues);
+    // Scans already built for this record hold the rows in storage; a row added now joins them
+    for (const [name, scanned] of this.scans) {
+      if (name.startsWith(`${this.entity}\u0000`)) {
+        (await scanned)?.add(entityValues);
+      }
+    }
+    this.record.entries.push(fileName);
+    for (const [key, buffer] of this.buffers) {
+      buffer.add((0, _computeKeyValue.computeKeyValue)(key.fields, entityValues), fileName);
+    }
+  }
+  /**
+   * Write the record file and each index once, with every row the statement added.
+   */
+  async write() {
+    await (0, _storage.writeJson)(`${this.record.path}.json`, this.record);
+    if (this.buffers.size) {
+      await (0, _storage.makeDirectory)(`__indexes/${this.record.path}`);
+    }
+    for (const [key, buffer] of this.buffers) {
+      await (0, _storage.writeJson)(`__indexes/${this.record.path}/${key.lookup}`, buffer.entries);
+    }
+  }
+}
+/**
+ * A new id for an autoGenerate field that no row holds yet, in storage or earlier in the statement.
+ * @param statement
+ * @param field
+ */
+const generateKey = async (statement, field) => {
+  let uuid;
+  do {
+    uuid = crypto.randomUUID();
+  } while (!(await statement.isUnique([field], {
+    [field]: uuid
+  })));
+  return uuid;
+};
 /**
  * Build one entity's stored values from the raw input values, generating any autoGenerate
- * fields and applying field defaults, then throw if a required field still has no value.
- * @param entity
- * @param record
+ * fields and applying field defaults, then throw if a required field still has no value. A null
+ * in an optional field that isn't nullable means the same as leaving it out (the field can't
+ * hold null, so nothing is lost): it isn't stored, and a default fills it in as for a field left out.
+ * @param statement
  * @param rowValues
  */
-const buildEntityValues = async (entity, record, rowValues = {}) => {
+const buildEntityValues = async (statement, rowValues = {}) => {
   const entityValues = {};
-  for (const field of record.definition) {
+  for (const field of statement.record.definition) {
     let value = rowValues[field.name];
+    if (value === null && field.optional && !field.nullable) {
+      value = undefined;
+    }
     if (field.autoGenerate) {
-      value = await (0, _keyGenerate.keyGenerate)(entity, [field.name]);
+      value = await generateKey(statement, field.name);
     } else if (Object.prototype.hasOwnProperty.call(field, 'default')) {
       value = (0, _useDefault.useDefault)(field['default'], value);
     }
     if (typeof value === 'undefined') {
       if (!field.optional) {
-        throw new Error(`Missing required field "${field.name}" for ${entity}`);
+        throw new Error(`Missing required field "${field.name}" for ${statement.entity}`);
       }
       continue;
     }
@@ -49,34 +344,30 @@ const buildEntityValues = async (entity, record, rowValues = {}) => {
  * Reject the insert if it collides with an existing primary or unique key value - a single field,
  * or a composite of several for a multi-field key. As in SQL, a value with a null in it is never a
  * duplicate, so a nullable unique field can hold null in any number of rows.
- * @param entity
- * @param record
+ * @param statement
  * @param entityValues
  */
-const assertUnique = async (entity, record, entityValues) => {
-  for (const key of record.keys) {
+const assertUnique = async (statement, entityValues) => {
+  for (const key of statement.record.keys) {
     if (key.type !== 'primary' && key.type !== 'unique') {
       continue;
     }
-    const checkKeys = key.fields.map(field => entityValues[field]);
-    if (checkKeys.some(value => value === null)) {
+    if (key.fields.some(field => entityValues[field] === null)) {
       continue;
     }
-    const isUnique = await (0, _keyUnique.keyUnique)(entity, key.fields, checkKeys);
-    if (!isUnique) {
-      throw new Error(`Duplicate value for ${key.type} key "${key.fields.join(', ')}" on ${entity}`);
+    if (!(await statement.isUnique(key.fields, entityValues))) {
+      throw new Error(`Duplicate value for ${key.type} key "${key.fields.join(', ')}" on ${statement.entity}`);
     }
   }
 };
 /**
  * Reject the insert if a single-field foreign key's value doesn't exist in the record it
  * references. A null references nothing, so it isn't checked (as in SQL).
- * @param entity
- * @param record
+ * @param statement
  * @param entityValues
  */
-const assertForeignKeys = async (entity, record, entityValues) => {
-  for (const key of record.keys) {
+const assertForeignKeys = async (statement, entityValues) => {
+  for (const key of statement.record.keys) {
     if (key.type !== 'foreign' || key.fields.length !== 1 || !key.references || key.references.length !== 1) {
       continue;
     }
@@ -88,12 +379,19 @@ const assertForeignKeys = async (entity, record, entityValues) => {
       entity: refEntity,
       property: refProperty
     } = (0, _splitEntityProperty.splitEntityProperty)(key.references[0].fields[0]);
-    const exists = await (0, _foreignKeyExists.foreignKeyExists)(refEntity, refProperty, entityValues[field]);
-    if (!exists) {
-      throw new Error(`Foreign key "${field}" on ${entity} references a nonexistent ${refEntity}.${refProperty} = ${entityValues[field]}`);
+    if (!(await statement.exists(refEntity, refProperty, entityValues[field]))) {
+      throw new Error(`Foreign key "${field}" on ${statement.entity} references a nonexistent ${refEntity}.${refProperty} = ${entityValues[field]}`);
     }
   }
 };
+/**
+ * Insert rows into a record, all in one statement: every row is checked (against the rows in storage and the rows
+ * before it in the statement) and its entry file written, then the record file and each index are written once.
+ * Inside a write set (as every query runs, see withWriteSet) a row that fails leaves none of them written.
+ * @param entity
+ * @param values - The rows, as field -> value.
+ * @param dataSet
+ */
 const insertEntity = async (entity = '', values = [], dataSet = {}) => {
   const record = await (0, _retrieveRecord.retrieveRecord)(entity);
   const primaryKey = record.keys.find(key => key.type === 'primary' && key.fields.length === 1);
@@ -103,19 +401,26 @@ const insertEntity = async (entity = '', values = [], dataSet = {}) => {
   // The record's folder can be missing even though the record exists: git doesn't keep empty folders, so a database
   // committed before its first insert comes back from a fresh clone without it
   await (0, _storage.makeDirectory)(record.path);
+  if (!values.length) {
+    return dataSet;
+  }
+  // A field the record doesn't define would otherwise be dropped without a word
   for (const rowValues of values) {
-    const entityValues = await buildEntityValues(entity, record, rowValues);
-    await assertUnique(entity, record, entityValues);
-    await assertForeignKeys(entity, record, entityValues);
+    (0, _knownFields.assertFieldsDefined)(entity, record, Object.keys(rowValues || {}), 'insert');
+  }
+  await (0, _packedTable.discardPacked)(record.path);
+  const statement = new InsertStatement(entity, record);
+  await statement.load();
+  for (const rowValues of values) {
+    const entityValues = await buildEntityValues(statement, rowValues);
+    await assertUnique(statement, entityValues);
+    await assertForeignKeys(statement, entityValues);
     const fileName = `${primaryKey ? entityValues[primaryKey.fields[0]] : crypto.randomUUID()}.json`;
     await (0, _storage.writeJson)(`${record.path}/${fileName}`, entityValues);
-    record.entries.push(fileName);
-    await (0, _storage.writeJson)(`${record.path}.json`, record);
-    for (const key of record.keys) {
-      await (0, _updateIndex.updateIndex)(record.path, key, (0, _computeKeyValue.computeKeyValue)(key.fields, entityValues), fileName, (0, _fieldType.keyValueType)(record, key));
-    }
+    await statement.add(entityValues, fileName);
     dataSet[entity].push(entityValues);
   }
+  await statement.write();
   return dataSet;
 };
 exports.insertEntity = insertEntity;

package/dist/data/queries/insertEntity.min.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.insertEntity=void 0,require("core-js/modules/esnext.iterator.constructor.js"),require("core-js/modules/esnext.iterator.find.js"),require("core-js/modules/esnext.iterator.map.js"),require("core-js/modules/esnext.iterator.some.js");var _retrieveRecord=require("../utilities/retrieveRecord"),_updateIndex=require("../utilities/updateIndex"),_keyGenerate=require("../introspect/policies/keyGenerate"),_keyUnique=require("../introspect/policies/keyUnique"),_useDefault=require("../introspect/policies/useDefault"),_typeCheck=require("../introspect/policies/typeCheck"),_foreignKeyExists=require("../introspect/policies/foreignKeyExists"),_splitEntityProperty=require("../utilities/parsers/splitEntityProperty"),_computeKeyValue=require("../utilities/computeKeyValue"),_fieldType=require("../utilities/fieldType"),_storage=require("../storage");const buildEntityValues=async(e,t,r={})=>{const i={};for(const s of t.definition){let t=r[s.name];if(s.autoGenerate?t=await(0,_keyGenerate.keyGenerate)(e,[s.name]):Object.prototype.hasOwnProperty.call(s,"default")&&(t=(0,_useDefault.useDefault)(s.default,t)),void 0!==t)i[s.name]=(0,_typeCheck.typeCheck)(s.type,t,s.nullable);else if(!s.optional)throw new Error(`Missing required field "${s.name}" for ${e}`)}return i},assertUnique=async(e,t,r)=>{for(const i of t.keys){if("primary"!==i.type&&"unique"!==i.type)continue;const t=i.fields.map((e=>r[e]));if(t.some((e=>null===e)))continue;if(!await(0,_keyUnique.keyUnique)(e,i.fields,t))throw new Error(`Duplicate value for ${i.type} key "${i.fields.join(", ")}" on ${e}`)}},assertForeignKeys=async(e,t,r)=>{for(const i of t.keys){if("foreign"!==i.type||1!==i.fields.length||!i.references||1!==i.references.length)continue;const t=i.fields[0];if(!Object.prototype.hasOwnProperty.call(r,t)||null===r[t])continue;const{entity:s,property:o}=(0,_splitEntityProperty.splitEntityProperty)(i.references[0].fields[0]);if(!await(0,_foreignKeyExists.foreignKeyExists)(s,o,r[t]))throw new Error(`Foreign key "${t}" on ${e} references a nonexistent ${s}.${o} = ${r[t]}`)}},insertEntity=async(e="",t=[],r={})=>{const i=await(0,_retrieveRecord.retrieveRecord)(e),s=i.keys.find((e=>"primary"===e.type&&1===e.fields.length));Object.prototype.hasOwnProperty.call(r,e)||(r[e]=[]),await(0,_storage.makeDirectory)(i.path);for(const o of t){const t=await buildEntityValues(e,i,o);await assertUnique(e,i,t),await assertForeignKeys(e,i,t);const n=`${s?t[s.fields[0]]:crypto.randomUUID()}.json`;await(0,_storage.writeJson)(`${i.path}/${n}`,t),i.entries.push(n),await(0,_storage.writeJson)(`${i.path}.json`,i);for(const e of i.keys)await(0,_updateIndex.updateIndex)(i.path,e,(0,_computeKeyValue.computeKeyValue)(e.fields,t),n,(0,_fieldType.keyValueType)(i,e));r[e].push(t)}return r};exports.insertEntity=insertEntity;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.insertEntity=void 0,require("core-js/modules/es.array.includes.js"),require("core-js/modules/es.json.stringify.js"),require("core-js/modules/esnext.iterator.constructor.js"),require("core-js/modules/esnext.iterator.every.js"),require("core-js/modules/esnext.iterator.find.js"),require("core-js/modules/esnext.iterator.for-each.js"),require("core-js/modules/esnext.iterator.map.js"),require("core-js/modules/esnext.iterator.some.js"),require("core-js/modules/esnext.map.delete-all.js"),require("core-js/modules/esnext.map.every.js"),require("core-js/modules/esnext.map.filter.js"),require("core-js/modules/esnext.map.find.js"),require("core-js/modules/esnext.map.find-key.js"),require("core-js/modules/esnext.map.includes.js"),require("core-js/modules/esnext.map.key-of.js"),require("core-js/modules/esnext.map.map-keys.js"),require("core-js/modules/esnext.map.map-values.js"),require("core-js/modules/esnext.map.merge.js"),require("core-js/modules/esnext.map.reduce.js"),require("core-js/modules/esnext.map.some.js"),require("core-js/modules/esnext.map.update.js"),require("core-js/modules/esnext.set.add-all.js"),require("core-js/modules/esnext.set.delete-all.js"),require("core-js/modules/esnext.set.difference.js"),require("core-js/modules/esnext.set.every.js"),require("core-js/modules/esnext.set.filter.js"),require("core-js/modules/esnext.set.find.js"),require("core-js/modules/esnext.set.intersection.js"),require("core-js/modules/esnext.set.is-disjoint-from.js"),require("core-js/modules/esnext.set.is-subset-of.js"),require("core-js/modules/esnext.set.is-superset-of.js"),require("core-js/modules/esnext.set.join.js"),require("core-js/modules/esnext.set.map.js"),require("core-js/modules/esnext.set.reduce.js"),require("core-js/modules/esnext.set.some.js"),require("core-js/modules/esnext.set.symmetric-difference.js"),require("core-js/modules/esnext.set.union.js");var _retrieveRecord=require("../utilities/retrieveRecord"),_keyUnique=require("../introspect/policies/keyUnique"),_useDefault=require("../introspect/policies/useDefault"),_typeCheck=require("../introspect/policies/typeCheck"),_foreignKeyExists=require("../introspect/policies/foreignKeyExists"),_splitEntityProperty=require("../utilities/parsers/splitEntityProperty"),_computeKeyValue=require("../utilities/computeKeyValue"),_fieldType=require("../utilities/fieldType"),_compareAs=require("../utilities/compareAs"),_retrieveRecords=require("../utilities/retrieveRecords"),_isSingleFieldKey=require("../utilities/isSingleFieldKey"),_indexBuffer=require("../utilities/indexBuffer"),_storage=require("../storage"),_packedTable=require("../utilities/packedTable"),_knownFields=require("../utilities/knownFields");class RowValues{constructor(e,s){this.seen=new Set,this.fields=s,this.types=s.map((s=>(0,_fieldType.fieldType)(e,s)))}static usable(e,s){return s.every((s=>_compareAs.comparableTypes.includes((0,_fieldType.fieldType)(e,s))))}keyOf(e){const s=[];for(let t=0;t<this.fields.length;t++){const r=e[this.fields[t]];if(null==r)return;const i=(0,_compareAs.compareAs)(this.types[t],r);if("object"==typeof i)return;s.push([typeof i,i])}return JSON.stringify(s)}add(e){const s=this.keyOf(e);void 0!==s&&this.seen.add(s)}has(e){const s=this.keyOf(e);return void 0!==s&&this.seen.has(s)}}class InsertStatement{constructor(e,s){this.rows=[],this.buffers=new Map,this.unreadable=new Set,this.otherLookups=new Map,this.scans=new Map,this.entity=e,this.record=s,this.hadEntries=(s.entries||[]).length>0}async load(){for(const e of this.record.keys){if(!e.lookup)continue;let s=[];try{s=await(0,_storage.readJson)(`__indexes/${this.record.path}/${e.lookup}`)}catch(s){this.unreadable.add(e)}this.buffers.set(e,new _indexBuffer.IndexBuffer(Array.isArray(s)?s:[],(0,_fieldType.keyValueType)(this.record,e)))}}ownLookup(e){const s=this.record.keys.find((s=>(0,_isSingleFieldKey.isSingleFieldKey)(s,e)&&s.lookup));if(!s||this.unreadable.has(s))return()=>{};const t=this.buffers.get(s);return e=>(this.hadEntries||this.rows.length>0)&&t.has(e)}otherLookup(e,s){const t=`${e}\0${s}`;return this.otherLookups.has(t)||this.otherLookups.set(t,(async()=>{const t=await(0,_retrieveRecord.retrieveRecord)(e),r=t.keys.find((e=>(0,_isSingleFieldKey.isSingleFieldKey)(e,s)&&e.lookup));if(!r)return()=>{};if(!(t.entries||[]).length)return()=>!1;let i;try{i=await(0,_storage.readJson)(`__indexes/${t.path}/${r.lookup}`)}catch(e){return()=>{}}const o=new _indexBuffer.IndexBuffer(Array.isArray(i)?i:[],(0,_fieldType.fieldType)(t,s));return e=>o.has(e)})()),this.otherLookups.get(t)}scan(e,s){const t=`${e}\0${s.join("\0")}`;return this.scans.has(t)||this.scans.set(t,(async()=>{const t=e===this.entity?this.record:await(0,_retrieveRecord.retrieveRecord)(e);if(!RowValues.usable(t,s))return;const r=new RowValues(t,s);for(const s of await(0,_retrieveRecords.retrieveRecords)(e))r.add(s);return e===this.entity&&this.rows.forEach((e=>r.add(e))),r})()),this.scans.get(t)}claimed(e,s){return this.rows.some((t=>e.every((e=>null!==t[e]&&void 0!==t[e]&&(0,_compareAs.sameValue)((0,_fieldType.fieldType)(this.record,e),t[e],s[e])))))}async isUnique(e,s){if(1===e.length){const t=this.ownLookup(e[0])(s[e[0]]);if(void 0!==t)return!t}const t=await this.scan(this.entity,e);return t?!t.has(s):await(0,_keyUnique.keyUnique)(this.entity,e,e.map((e=>s[e])))&&!this.claimed(e,s)}async exists(e,s,t){const r=e===this.entity,i=r?this.ownLookup(s)(t):(await this.otherLookup(e,s))(t);if(void 0!==i)return i;const o=await this.scan(e,[s]);return o?o.has({[s]:t}):await(0,_foreignKeyExists.foreignKeyExists)(e,s,t)||r&&this.claimed([s],{[s]:t})}async add(e,s){this.rows.push(e);for(const[s,t]of this.scans)s.startsWith(`${this.entity}\0`)&&(await t)?.add(e);this.record.entries.push(s);for(const[t,r]of this.buffers)r.add((0,_computeKeyValue.computeKeyValue)(t.fields,e),s)}async write(){await(0,_storage.writeJson)(`${this.record.path}.json`,this.record),this.buffers.size&&await(0,_storage.makeDirectory)(`__indexes/${this.record.path}`);for(const[e,s]of this.buffers)await(0,_storage.writeJson)(`__indexes/${this.record.path}/${e.lookup}`,s.entries)}}const generateKey=async(e,s)=>{let t;do{t=crypto.randomUUID()}while(!await e.isUnique([s],{[s]:t}));return t},buildEntityValues=async(e,s={})=>{const t={};for(const r of e.record.definition){let i=s[r.name];if(null===i&&r.optional&&!r.nullable&&(i=void 0),r.autoGenerate?i=await generateKey(e,r.name):Object.prototype.hasOwnProperty.call(r,"default")&&(i=(0,_useDefault.useDefault)(r.default,i)),void 0!==i)t[r.name]=(0,_typeCheck.typeCheck)(r.type,i,r.nullable);else if(!r.optional)throw new Error(`Missing required field "${r.name}" for ${e.entity}`)}return t},assertUnique=async(e,s)=>{for(const t of e.record.keys)if(!("primary"!==t.type&&"unique"!==t.type||t.fields.some((e=>null===s[e]))||await e.isUnique(t.fields,s)))throw new Error(`Duplicate value for ${t.type} key "${t.fields.join(", ")}" on ${e.entity}`)},assertForeignKeys=async(e,s)=>{for(const t of e.record.keys){if("foreign"!==t.type||1!==t.fields.length||!t.references||1!==t.references.length)continue;const r=t.fields[0];if(!Object.prototype.hasOwnProperty.call(s,r)||null===s[r])continue;const{entity:i,property:o}=(0,_splitEntityProperty.splitEntityProperty)(t.references[0].fields[0]);if(!await e.exists(i,o,s[r]))throw new Error(`Foreign key "${r}" on ${e.entity} references a nonexistent ${i}.${o} = ${s[r]}`)}},insertEntity=async(e="",s=[],t={})=>{const r=await(0,_retrieveRecord.retrieveRecord)(e),i=r.keys.find((e=>"primary"===e.type&&1===e.fields.length));if(Object.prototype.hasOwnProperty.call(t,e)||(t[e]=[]),await(0,_storage.makeDirectory)(r.path),!s.length)return t;for(const t of s)(0,_knownFields.assertFieldsDefined)(e,r,Object.keys(t||{}),"insert");await(0,_packedTable.discardPacked)(r.path);const o=new InsertStatement(e,r);await o.load();for(const n of s){const s=await buildEntityValues(o,n);await assertUnique(o,s),await assertForeignKeys(o,s);const a=`${i?s[i.fields[0]]:crypto.randomUUID()}.json`;await(0,_storage.writeJson)(`${r.path}/${a}`,s),await o.add(s,a),t[e].push(s)}return await o.write(),t};exports.insertEntity=insertEntity;

package/dist/data/queries/query.d.ts

@@ -1,7 +1,8 @@
 /**
  * Parse a query string (one query, or several separated by `;`) and run each query in turn, resolving to one result
  * per query. The queries of one call run together, under the database lock: no other call's queries run between
- * them (see withDatabaseLock).
+ * them (see withDatabaseLock). Each query is all or nothing: one that fails changes nothing, and the queries before it
+ * keep their changes (see withWriteSet).
  * @param queryString
  */
 export declare const query: (queryString?: string) => Promise<any[]>;

package/dist/data/queries/query.js

@@ -7,10 +7,12 @@ exports.query = void 0;
 var _parser = require("./parser");
 var _runQuery = require("../utilities/runQuery");
 var _databaseLock = require("../utilities/databaseLock");
+var _storage = require("../storage");
 /**
  * Parse a query string (one query, or several separated by `;`) and run each query in turn, resolving to one result
  * per query. The queries of one call run together, under the database lock: no other call's queries run between
- * them (see withDatabaseLock).
+ * them (see withDatabaseLock). Each query is all or nothing: one that fails changes nothing, and the queries before it
+ * keep their changes (see withWriteSet).
  * @param queryString
  */
 const query = async (queryString = '') => {
@@ -18,7 +20,8 @@ const query = async (queryString = '') => {
   return (0, _databaseLock.withDatabaseLock)(async () => {
     const queryResults = [];
     for (const parsed of parsedQueries) {
-      const dataSet = await (0, _runQuery.runQuery)(parsed);
+      // Each query's changes reach storage together, or not at all if it fails (see withWriteSet)
+      const dataSet = await (0, _storage.withWriteSet)(() => (0, _runQuery.runQuery)(parsed));
       queryResults.push(dataSet[parsed.entity]);
     }
     return queryResults;

package/dist/data/queries/query.min.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.query=void 0;var _parser=require("./parser"),_runQuery=require("../utilities/runQuery"),_databaseLock=require("../utilities/databaseLock");const query=async(e="")=>{const r=(0,_parser.parser)(e);return(0,_databaseLock.withDatabaseLock)((async()=>{const e=[];for(const t of r){const r=await(0,_runQuery.runQuery)(t);e.push(r[t.entity])}return e}))};exports.query=query;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.query=void 0;var _parser=require("./parser"),_runQuery=require("../utilities/runQuery"),_databaseLock=require("../utilities/databaseLock"),_storage=require("../storage");const query=async(e="")=>{const r=(0,_parser.parser)(e);return(0,_databaseLock.withDatabaseLock)((async()=>{const e=[];for(const t of r){const r=await(0,_storage.withWriteSet)((()=>(0,_runQuery.runQuery)(t)));e.push(r[t.entity])}return e}))};exports.query=query;

package/dist/data/queries/selectFields.d.ts

@@ -1,12 +1,13 @@
 import { ParsedSelect } from '../utilities/parsers/findSelects';
 import { Group } from './groupBy';
 /**
- * Shape each row down to only the selected properties (aliased, when given). dataSet is either a list of rows, or
- * (once groupBy has run) a list of groups, whose rows are shaped the same way at any depth; the groups keep their
- * field and value, selected or not.
+ * Shape each row down to only the selected properties (aliased, when given, otherwise named as written). dataSet is
+ * either a list of rows, or (once groupBy has run) a list of groups, whose rows are shaped the same way at any depth;
+ * the groups keep their field and value, selected or not.
  * @param dataSet
  * @param selectClauses
  * @param grouped Whether dataSet is a list of groups (see groupBy)
+ * @param entity The queried record's name, so `<record>.<field>` picks the row's own field
  */
-export declare const selectFields: (dataSet?: Object[] | Group[], selectClauses?: ParsedSelect[], grouped?: boolean) => Object[] | Group[];
+export declare const selectFields: (dataSet?: Object[] | Group[], selectClauses?: ParsedSelect[], grouped?: boolean, entity?: string) => Object[] | Group[];
 //# sourceMappingURL=selectFields.d.ts.map

package/dist/data/queries/selectFields.js

@@ -7,35 +7,49 @@ exports.selectFields = void 0;
 require("core-js/modules/esnext.iterator.constructor.js");
 require("core-js/modules/esnext.iterator.map.js");
 require("core-js/modules/esnext.iterator.reduce.js");
-const shapeRow = (row, selectClauses) => selectClauses.reduce((shaped, {
+/**
+ * The value a select name picks from a row: the row's own key (a field, a merged record's `<record>.<field>`, or a
+ * joined record's rows), or for `<queried record>.<field>`, the row's own field.
+ * @param row
+ * @param property
+ * @param entity - The queried record's name.
+ */
+const pick = (row, property, entity) => {
+  if (Object.prototype.hasOwnProperty.call(row, property) || typeof entity === 'undefined' || !property.startsWith(`${entity}.`)) {
+    return row[property];
+  }
+  return row[property.slice(entity.length + 1)];
+};
+const shapeRow = (row, selectClauses, entity) => selectClauses.reduce((shaped, {
   property,
   alias
 }) => {
-  shaped[alias || property] = row[property];
+  shaped[alias || property] = pick(row, property, entity);
   return shaped;
 }, {});
-const shapeGroup = (group, selectClauses) => group.groups ? {
+const shapeGroup = (group, selectClauses, entity) => group.groups ? {
   ...group,
-  groups: group.groups.map(inner => shapeGroup(inner, selectClauses))
+  groups: group.groups.map(inner => shapeGroup(inner, selectClauses, entity))
 } : {
   ...group,
-  rows: group.rows.map(row => shapeRow(row, selectClauses))
+  rows: group.rows.map(row => shapeRow(row, selectClauses, entity))
 };
 /**
- * Shape each row down to only the selected properties (aliased, when given). dataSet is either a list of rows, or
- * (once groupBy has run) a list of groups, whose rows are shaped the same way at any depth; the groups keep their
- * field and value, selected or not.
+ * Shape each row down to only the selected properties (aliased, when given, otherwise named as written). dataSet is
+ * either a list of rows, or (once groupBy has run) a list of groups, whose rows are shaped the same way at any depth;
+ * the groups keep their field and value, selected or not.
  * @param dataSet
  * @param selectClauses
  * @param grouped Whether dataSet is a list of groups (see groupBy)
+ * @param entity The queried record's name, so `<record>.<field>` picks the row's own field
  */
-const selectFields = (dataSet = [], selectClauses = [], grouped = false) => {
+const selectFields = (dataSet = [], selectClauses = [], grouped = false, entity) => {
   if (!selectClauses.length) {
     return dataSet;
   }
   if (grouped) {
-    return dataSet.map(group => shapeGroup(group, selectClauses));
+    return dataSet.map(group => shapeGroup(group, selectClauses, entity));
   }
-  return dataSet.map(row => shapeRow(row, selectClauses));
+  return dataSet.map(row => shapeRow(row, selectClauses, entity));
 };
 exports.selectFields = selectFields;

package/dist/data/queries/selectFields.min.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.selectFields=void 0,require("core-js/modules/esnext.iterator.constructor.js"),require("core-js/modules/esnext.iterator.map.js"),require("core-js/modules/esnext.iterator.reduce.js");const shapeRow=(e,s)=>s.reduce(((s,{property:r,alias:o})=>(s[o||r]=e[r],s)),{}),shapeGroup=(e,s)=>e.groups?{...e,groups:e.groups.map((e=>shapeGroup(e,s)))}:{...e,rows:e.rows.map((e=>shapeRow(e,s)))},selectFields=(e=[],s=[],r=!1)=>s.length?r?e.map((e=>shapeGroup(e,s))):e.map((e=>shapeRow(e,s))):e;exports.selectFields=selectFields;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.selectFields=void 0,require("core-js/modules/esnext.iterator.constructor.js"),require("core-js/modules/esnext.iterator.map.js"),require("core-js/modules/esnext.iterator.reduce.js");const pick=(e,s,r)=>Object.prototype.hasOwnProperty.call(e,s)||void 0===r||!s.startsWith(`${r}.`)?e[s]:e[s.slice(r.length+1)],shapeRow=(e,s,r)=>s.reduce(((s,{property:o,alias:t})=>(s[t||o]=pick(e,o,r),s)),{}),shapeGroup=(e,s,r)=>e.groups?{...e,groups:e.groups.map((e=>shapeGroup(e,s,r)))}:{...e,rows:e.rows.map((e=>shapeRow(e,s,r)))},selectFields=(e=[],s=[],r=!1,o)=>s.length?r?e.map((e=>shapeGroup(e,s,o))):e.map((e=>shapeRow(e,s,o))):e;exports.selectFields=selectFields;