@absolutejs/sync 0.4.0 → 0.6.0

package/dist/engine/index.js

@@ -885,6 +885,158 @@ var query = (source) => makeQuery(source, []);
 var defineGraphCollection = (definition) => ({ ...definition, kind: "graph" });
 // src/engine/permissions.ts
 var definePermissions = (permissions) => permissions;
+// src/engine/search.ts
+var SEARCH_SCORE_FIELD = "_score";
+var defineSearchCollection = (definition) => ({
+  ...definition,
+  kind: "search"
+});
+// src/engine/textIndex.ts
+var defaultTokenize = (text) => text.toLowerCase().match(/[a-z0-9]+/g) ?? [];
+var createTextIndex = (options) => {
+  const { key, fields } = options;
+  const tokenize = options.tokenize ?? defaultTokenize;
+  const stopwords = new Set(options.stopwords ?? []);
+  const k1 = options.k1 ?? 1.5;
+  const b = options.b ?? 0.75;
+  const docs = new Map;
+  const postings = new Map;
+  let totalLen = 0;
+  const termsOf = (row) => {
+    const text = fields.map((field) => {
+      const value = row[field];
+      return value === undefined || value === null ? "" : String(value);
+    }).join(" ");
+    return tokenize(text).filter((term) => !stopwords.has(term));
+  };
+  const remove = (rowKey) => {
+    const doc = docs.get(rowKey);
+    if (doc === undefined) {
+      return;
+    }
+    for (const term of doc.tf.keys()) {
+      const set = postings.get(term);
+      if (set !== undefined) {
+        set.delete(rowKey);
+        if (set.size === 0) {
+          postings.delete(term);
+        }
+      }
+    }
+    totalLen -= doc.len;
+    docs.delete(rowKey);
+  };
+  const add = (row) => {
+    const rowKey = key(row);
+    remove(rowKey);
+    const terms = termsOf(row);
+    const tf = new Map;
+    for (const term of terms) {
+      tf.set(term, (tf.get(term) ?? 0) + 1);
+    }
+    for (const term of tf.keys()) {
+      let set = postings.get(term);
+      if (set === undefined) {
+        set = new Set;
+        postings.set(term, set);
+      }
+      set.add(rowKey);
+    }
+    docs.set(rowKey, { row, len: terms.length, tf });
+    totalLen += terms.length;
+  };
+  const search = (query2, limit) => {
+    const total = docs.size;
+    if (total === 0) {
+      return [];
+    }
+    const avgdl = totalLen / total;
+    const queryTerms = new Set(tokenize(query2).filter((term) => !stopwords.has(term)));
+    const scores = new Map;
+    for (const term of queryTerms) {
+      const set = postings.get(term);
+      if (set === undefined) {
+        continue;
+      }
+      const df = set.size;
+      const idf = Math.log(1 + (total - df + 0.5) / (df + 0.5));
+      for (const rowKey of set) {
+        const doc = docs.get(rowKey);
+        const freq = doc.tf.get(term) ?? 0;
+        const norm = freq * (k1 + 1) / (freq + k1 * (1 - b + b * doc.len / avgdl));
+        scores.set(rowKey, (scores.get(rowKey) ?? 0) + idf * norm);
+      }
+    }
+    return [...scores.entries()].map(([rowKey, score]) => ({ row: docs.get(rowKey).row, score })).sort((first, second) => second.score - first.score).slice(0, limit);
+  };
+  return {
+    add,
+    remove,
+    search,
+    size: () => docs.size,
+    clear: () => {
+      docs.clear();
+      postings.clear();
+      totalLen = 0;
+    }
+  };
+};
+// src/engine/vectorIndex.ts
+var dot = (first, second) => {
+  const length = Math.min(first.length, second.length);
+  let sum = 0;
+  for (let index = 0;index < length; index += 1) {
+    sum += first[index] * second[index];
+  }
+  return sum;
+};
+var normOf = (vec) => Math.sqrt(dot(vec, vec));
+var euclidean = (first, second) => {
+  const length = Math.max(first.length, second.length);
+  let sum = 0;
+  for (let index = 0;index < length; index += 1) {
+    const delta = (first[index] ?? 0) - (second[index] ?? 0);
+    sum += delta * delta;
+  }
+  return Math.sqrt(sum);
+};
+var createVectorIndex = (options) => {
+  const { key, embedding } = options;
+  const metric = options.metric ?? "cosine";
+  const entries = new Map;
+  const score = (query2, queryNorm, entry) => {
+    if (metric === "dot") {
+      return dot(query2, entry.vec);
+    }
+    if (metric === "euclidean") {
+      return -euclidean(query2, entry.vec);
+    }
+    const denominator = queryNorm * entry.norm;
+    return denominator === 0 ? 0 : dot(query2, entry.vec) / denominator;
+  };
+  return {
+    add: (row) => {
+      const vec = embedding(row);
+      entries.set(key(row), { row, vec, norm: normOf(vec) });
+    },
+    remove: (rowKey) => {
+      entries.delete(rowKey);
+    },
+    search: (query2, limit) => {
+      const queryNorm = normOf(query2);
+      return [...entries.values()].map((entry) => ({
+        row: entry.row,
+        score: score(query2, queryNorm, entry)
+      })).sort((first, second) => second.score - first.score).slice(0, limit);
+    },
+    size: () => entries.size,
+    clear: () => {
+      entries.clear();
+    }
+  };
+};
+// src/engine/schedule.ts
+var defineSchedule = (definition) => definition;
 // src/engine/mutation.ts
 var defineMutation = (definition) => definition;
 // src/engine/syncEngine.ts
@@ -906,11 +1058,21 @@ var shallowEqual4 = (a, b) => {
   const bKeys = Object.keys(b);
   return aKeys.length === bKeys.length && aKeys.every((k) => a[k] === b[k]);
 };
+var equalsIgnoringScore = (a, b) => {
+  if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) {
+    return a === b;
+  }
+  const strip = (value) => Object.keys(value).filter((k) => k !== SEARCH_SCORE_FIELD);
+  const aKeys = strip(a);
+  const bKeys = strip(b);
+  return aKeys.length === bKeys.length && aKeys.every((k) => a[k] === b[k]);
+};
 var createSyncEngine = (options = {}) => {
   const registry = new Map;
   const mutations = new Map;
   const writers = new Map;
   const readers = new Map;
+  const schedules = new Map;
   const permissions = new Map;
   for (const [table, rules] of Object.entries(options.permissions ?? {})) {
     permissions.set(table, rules);
@@ -921,6 +1083,8 @@ var createSyncEngine = (options = {}) => {
     return rules?.[op] ?? rules?.write;
   };
   const reactiveSubs = new Set;
+  const searchSubs = new Set;
+  const searchIndexes = new Map;
   const active = new Map;
   const tableIndex = new Map;
   const changeLogSize = options.changeLogSize ?? 1024;
@@ -973,6 +1137,9 @@ var createSyncEngine = (options = {}) => {
     if (subscription.kind === "reactive") {
       return EMPTY_DIFF;
     }
+    if (subscription.kind === "search") {
+      return EMPTY_DIFF;
+    }
     if (subscription.incremental) {
       try {
         return subscription.view.apply(change);
@@ -1037,7 +1204,7 @@ var createSyncEngine = (options = {}) => {
   };
   const depKey = (table, key) => `${table} ${key}`;
   const changedKeyFor = (table, change) => readers.get(table)?.key?.(change.row);
-  const makeReadHandle = (ctx, readTables, readKeys, rangeDeps) => {
+  const makeReadHandle = (ctx, readTables, readKeys, rangeDeps, applyRules = true) => {
     const readerFor = (table) => {
       const reader = readers.get(table);
       if (reader === undefined) {
@@ -1045,11 +1212,12 @@ var createSyncEngine = (options = {}) => {
       }
       return reader;
     };
+    const ruleFor = (table) => applyRules ? readRuleFor(table) : undefined;
     return {
       all: async (table) => {
         readTables.add(table);
         const rows = [...await readerFor(table).all(ctx)];
-        const rule = readRuleFor(table);
+        const rule = ruleFor(table);
         return rule ? rows.filter((row) => rule(ctx, row)) : rows;
       },
       get: async (table, key) => {
@@ -1063,12 +1231,12 @@ var createSyncEngine = (options = {}) => {
           readTables.add(table);
         }
         const row = await reader.get(key, ctx);
-        const rule = readRuleFor(table);
+        const rule = ruleFor(table);
         return rule && row !== undefined && !rule(ctx, row) ? undefined : row;
       },
       where: async (table, predicate) => {
         const reader = readerFor(table);
-        const rule = readRuleFor(table);
+        const rule = ruleFor(table);
         const effective = rule ? (row) => predicate(row) && rule(ctx, row) : predicate;
         const matched = [...await reader.all(ctx)].filter(effective);
         if (reader.key !== undefined) {
@@ -1085,7 +1253,72 @@ var createSyncEngine = (options = {}) => {
       }
     };
   };
-  const diffRerun = (sub, rows) => {
+  const writerFor = (table) => {
+    const writer = writers.get(table);
+    if (writer === undefined) {
+      throw new Error(`No writer registered for table "${table}" \u2014 register one with engine.registerWriter, or use actions.change`);
+    }
+    return writer;
+  };
+  const authorizeWrite = async (table, op, value, ctx) => {
+    const rule = writeRuleFor(table, op);
+    if (rule === undefined) {
+      return;
+    }
+    let subject = value;
+    if (op !== "insert") {
+      const reader = readers.get(table);
+      if (reader?.get !== undefined) {
+        const id = reader.key ? reader.key(value) : value.id;
+        if (id !== undefined) {
+          const existing = await reader.get(id, ctx);
+          if (existing !== undefined) {
+            subject = existing;
+          }
+        }
+      }
+    }
+    if (!rule(ctx, subject)) {
+      throw new UnauthorizedError(`${op} on table "${table}"`);
+    }
+  };
+  const makeActions = (tx, ctx, enforce) => {
+    const buffered = [];
+    const actions = {
+      change: (collection, change) => {
+        buffered.push({
+          table: collection,
+          change
+        });
+        return Promise.resolve();
+      },
+      insert: async (table, data) => {
+        if (enforce) {
+          await authorizeWrite(table, "insert", data, ctx);
+        }
+        const row = await writerFor(table).insert(data, ctx, tx);
+        buffered.push({ table, change: { op: "insert", row } });
+        return row;
+      },
+      update: async (table, data) => {
+        if (enforce) {
+          await authorizeWrite(table, "update", data, ctx);
+        }
+        const row = await writerFor(table).update(data, ctx, tx);
+        buffered.push({ table, change: { op: "update", row } });
+        return row;
+      },
+      delete: async (table, row) => {
+        if (enforce) {
+          await authorizeWrite(table, "delete", row, ctx);
+        }
+        await writerFor(table).delete(row, ctx, tx);
+        buffered.push({ table, change: { op: "delete", row } });
+      }
+    };
+    return { actions, buffered };
+  };
+  const diffRerun = (sub, rows, equals = shallowEqual4) => {
     const next = new Map;
     for (const row of rows) {
       next.set(sub.key(row), row);
@@ -1097,7 +1330,7 @@ var createSyncEngine = (options = {}) => {
       const previous = sub.current.get(rowKey);
       if (previous === undefined) {
         added.push(row);
-      } else if (!shallowEqual4(previous, row)) {
+      } else if (!equals(previous, row)) {
         changed.push(row);
       }
     }
@@ -1128,6 +1361,47 @@ var createSyncEngine = (options = {}) => {
     }
     return pairs;
   };
+  const ensureSearchIndex = async (definition) => {
+    let entry = searchIndexes.get(definition.name);
+    if (entry === undefined) {
+      entry = { index: definition.index(), definition, hydrated: false };
+      searchIndexes.set(definition.name, entry);
+    }
+    if (!entry.hydrated) {
+      for (const row of await definition.source()) {
+        entry.index.add(row);
+      }
+      entry.hydrated = true;
+    }
+    return entry;
+  };
+  const searchPairs = (changes) => {
+    const touched = new Set;
+    for (const { table, change } of changes) {
+      for (const entry of searchIndexes.values()) {
+        if (!entry.hydrated || entry.definition.table !== table) {
+          continue;
+        }
+        if (change.op === "delete") {
+          entry.index.remove(entry.definition.key(change.row));
+        } else {
+          entry.index.add(change.row);
+        }
+        touched.add(entry.definition.name);
+      }
+    }
+    const pairs = [];
+    for (const sub of searchSubs) {
+      if (!touched.has(sub.collection)) {
+        continue;
+      }
+      const diff = diffRerun(sub, sub.rerun(), equalsIgnoringScore);
+      if (!isEmptyViewDiff(diff)) {
+        pairs.push([sub, diff]);
+      }
+    }
+    return pairs;
+  };
   const logChange = (changeVersion, entry) => {
     changeLog.push(entry);
     if (changeLog.length > changeLogSize) {
@@ -1148,6 +1422,7 @@ var createSyncEngine = (options = {}) => {
     emissions.push(...await reactivePairs([
       { table, key: changedKeyFor(table, change), row: change.row }
     ]));
+    emissions.push(...searchPairs([{ table, change }]));
     for (const [subscription, diff] of emissions) {
       subscription.onDiff(diff, changeVersion);
     }
@@ -1188,6 +1463,7 @@ var createSyncEngine = (options = {}) => {
       }
     }
     emissions.push(...await reactivePairs(reactiveChanges));
+    emissions.push(...searchPairs(changes));
     for (const [subscription, diff] of emissions) {
       subscription.onDiff(diff, batchVersion);
     }
@@ -1330,6 +1606,50 @@ var createSyncEngine = (options = {}) => {
       }
     };
   };
+  const subscribeSearch = async (collection, definition, params, ctx, onDiff, set) => {
+    const query2 = params;
+    if (definition.authorize !== undefined) {
+      const allowed = await definition.authorize(query2, ctx);
+      if (!allowed) {
+        throw new UnauthorizedError(`subscribe to collection "${collection}"`);
+      }
+    }
+    const entry = await ensureSearchIndex(definition);
+    const limit = definition.limit ?? 20;
+    const readRule = readRuleFor(definition.table);
+    const rerun = () => {
+      const candidates = entry.index.search(query2, readRule ? limit * 5 : limit);
+      const visible = readRule ? candidates.filter((hit) => readRule(ctx, hit.row)) : candidates;
+      return visible.slice(0, limit).map((hit) => ({
+        ...hit.row,
+        [SEARCH_SCORE_FIELD]: hit.score
+      }));
+    };
+    const initial = rerun();
+    const current = new Map;
+    for (const row of initial) {
+      current.set(definition.key(row), row);
+    }
+    const atVersion = version;
+    const subscription = {
+      kind: "search",
+      collection,
+      key: definition.key,
+      rerun,
+      current,
+      onDiff
+    };
+    set.add(subscription);
+    searchSubs.add(subscription);
+    return {
+      initial,
+      version: atVersion,
+      unsubscribe: () => {
+        set.delete(subscription);
+        searchSubs.delete(subscription);
+      }
+    };
+  };
   return {
     register: (collection) => {
       registry.set(collection.name, collection);
@@ -1348,6 +1668,9 @@ var createSyncEngine = (options = {}) => {
         addTableIndex(table, collection.name);
       }
     },
+    registerSearch: (collection) => {
+      registry.set(collection.name, collection);
+    },
     subscribe: async ({ collection, params, ctx, onDiff, since }) => {
       const registered = registry.get(collection);
       if (registered === undefined) {
@@ -1368,6 +1691,10 @@ var createSyncEngine = (options = {}) => {
         const reactived = await subscribeReactive(collection, registered, params, ctx, typedOnDiff, subscribeSet);
         return reactived;
       }
+      if (registeredKind === "search") {
+        const searched = await subscribeSearch(collection, registered, params, ctx, typedOnDiff, subscribeSet);
+        return searched;
+      }
       const definition = registered;
       if (definition.authorize !== undefined) {
         const allowed = await definition.authorize(params, ctx);
@@ -1488,69 +1815,32 @@ var createSyncEngine = (options = {}) => {
           throw new UnauthorizedError(`run mutation "${name}"`);
         }
       }
-      const writerFor = (table) => {
-        const writer = writers.get(table);
-        if (writer === undefined) {
-          throw new Error(`No writer registered for table "${table}" \u2014 register one with engine.registerWriter, or use actions.change`);
-        }
-        return writer;
-      };
-      const authorizeWrite = async (table, op, value) => {
-        const rule = writeRuleFor(table, op);
-        if (rule === undefined) {
-          return;
-        }
-        let subject = value;
-        if (op !== "insert") {
-          const reader = readers.get(table);
-          if (reader?.get !== undefined) {
-            const id = reader.key ? reader.key(value) : value.id;
-            if (id !== undefined) {
-              const existing = await reader.get(id, ctx);
-              if (existing !== undefined) {
-                subject = existing;
-              }
-            }
-          }
-        }
-        if (!rule(ctx, subject)) {
-          throw new UnauthorizedError(`${op} on table "${table}"`);
-        }
-      };
       const runHandler = async (tx) => {
-        const buffered2 = [];
-        const actions = {
-          change: (collection, change) => {
-            buffered2.push({
-              table: collection,
-              change
-            });
-            return Promise.resolve();
-          },
-          insert: async (table, data) => {
-            await authorizeWrite(table, "insert", data);
-            const row = await writerFor(table).insert(data, ctx, tx);
-            buffered2.push({ table, change: { op: "insert", row } });
-            return row;
-          },
-          update: async (table, data) => {
-            await authorizeWrite(table, "update", data);
-            const row = await writerFor(table).update(data, ctx, tx);
-            buffered2.push({ table, change: { op: "update", row } });
-            return row;
-          },
-          delete: async (table, row) => {
-            await authorizeWrite(table, "delete", row);
-            await writerFor(table).delete(row, ctx, tx);
-            buffered2.push({ table, change: { op: "delete", row } });
-          }
-        };
-        const handlerResult = await mutation.handler(args, ctx, actions);
-        return { buffered: buffered2, result: handlerResult };
+        const { actions, buffered: buffered2 } = makeActions(tx, ctx, true);
+        const result2 = await mutation.handler(args, ctx, actions);
+        return { buffered: buffered2, result: result2 };
       };
       const { buffered, result } = runInTransaction !== undefined ? await runInTransaction((tx) => runHandler(tx)) : await runHandler(undefined);
       await applyChangeBatch(buffered);
       return result;
+    },
+    registerSchedule: (schedule) => {
+      schedules.set(schedule.name, schedule);
+    },
+    listSchedules: () => [...schedules.values()],
+    runSchedule: async (name) => {
+      const schedule = schedules.get(name);
+      if (schedule === undefined) {
+        throw new Error(`Unknown schedule "${name}"`);
+      }
+      const runHandler = async (tx) => {
+        const { actions, buffered: buffered2 } = makeActions(tx, {}, false);
+        const db = makeReadHandle({}, new Set, new Set, [], false);
+        await schedule.run({ actions, db });
+        return buffered2;
+      };
+      const buffered = runInTransaction !== undefined ? await runInTransaction((tx) => runHandler(tx)) : await runHandler(undefined);
+      await applyChangeBatch(buffered);
     }
   };
 };
@@ -1801,12 +2091,16 @@ export {
   hydrateRoute,
   fromRowChange,
   filterOp,
+  defineSearchCollection,
+  defineSchedule,
   defineReactiveQuery,
   definePermissions,
   defineMutation,
   defineJoinCollection,
   defineGraphCollection,
   defineCollection,
+  createVectorIndex,
+  createTextIndex,
   createSyncEngine,
   createSyncConnection,
   createPresenceHub,
@@ -1817,8 +2111,9 @@ export {
   createAggregate,
   chain,
   aggregateOp,
-  UnauthorizedError
+  UnauthorizedError,
+  SEARCH_SCORE_FIELD
 };
 
-//# debugId=04C93A0AB37B69AD64756E2164756E21
+//# debugId=54AD7964887E323764756E2164756E21
 //# sourceMappingURL=index.js.map