@objy/server 0.0.0-stage → 0.1.0

package/src/plugin-loader.js

@@ -0,0 +1,21 @@
+import { pathToFileURL } from "node:url";
+import { dirname, isAbsolute, resolve } from "node:path";
+
+export async function loadPlugins(plugins, { configPath, ...context } = {}) {
+  if (!Array.isArray(plugins)) throw new TypeError("plugins must be an array");
+  const base = configPath ? dirname(configPath) : process.cwd();
+
+  for (const entry of plugins) {
+    const descriptor = typeof entry === "string" ? { file: entry } : entry;
+    if (!descriptor || typeof descriptor.file !== "string" || !descriptor.file.trim()) {
+      throw new TypeError("plugin must be a file path or { file, options }");
+    }
+    const file = isAbsolute(descriptor.file) ? descriptor.file : resolve(base, descriptor.file);
+    const module = await import(pathToFileURL(file).href);
+    const register = module.register || module.default;
+    if (typeof register !== "function") {
+      throw new TypeError(`plugin ${descriptor.file} must export register()`);
+    }
+    await register({ ...context, configPath, options: descriptor.options || {} });
+  }
+}

package/src/query-planner.js

@@ -0,0 +1,266 @@
+import { normalizeAggregateSpec } from "./aggregate.js";
+
+export const READ_PATH = {
+  INDEX_EXACT: "index-exact",
+  INDEX_PARTIAL: "index-partial",
+  ORDERED_INDEX: "ordered-index",
+  SCAN: "scan",
+};
+
+export const QUERY_PATH_HEADER = "x-objydb-query-path";
+
+export const AGGREGATE_PATH = {
+  COLUMN_SUMMARY: "aggregate-column-summary",
+  COLUMN_SCAN: "aggregate-column-scan",
+  GROUP_FORWARD_INDEX: "aggregate-group-forward-index",
+  GROUP_INDEX: "aggregate-group-index",
+  GROUP_COLUMN_SCAN: "aggregate-group-column-scan",
+  SCAN: "aggregate-scan",
+};
+
+export function publicReadPath(path) {
+  switch (path) {
+    case READ_PATH.INDEX_EXACT:
+      return "index-full";
+    case READ_PATH.INDEX_PARTIAL:
+      return "index-partial";
+    case READ_PATH.ORDERED_INDEX:
+      return "ordered-index";
+    case READ_PATH.SCAN:
+      return "scan";
+    default:
+      return path ? String(path) : null;
+  }
+}
+
+export function publicAggregatePath(path) {
+  return path ? String(path) : null;
+}
+
+export function combineQueryPaths(paths = []) {
+  const out = [];
+  const seen = new Set();
+  for (const path of paths) {
+    for (const part of String(path || "").split(",")) {
+      const value = part.trim();
+      if (!value || seen.has(value)) continue;
+      seen.add(value);
+      out.push(value);
+    }
+  }
+  return out.join(",");
+}
+
+export function planRead(criteria = {}, options = {}, adapter) {
+  const hasCriteria = hasObjectCriteria(criteria);
+  const idPlan = hasCriteria ? planIndexedIds(criteria, adapter) : allIdsPlan();
+
+  if (options.sortBy && adapter.hasOrderedIndex(options.sortBy)) {
+    return {
+      path: READ_PATH.ORDERED_INDEX,
+      idPlan,
+      sortBy: String(options.sortBy),
+      reason: "sort field has an ordered field index",
+    };
+  }
+
+  if (!hasCriteria) {
+    return {
+      path: READ_PATH.SCAN,
+      idPlan,
+      reason: "no criteria",
+    };
+  }
+
+  if (idPlan?.exact) {
+    return {
+      path: READ_PATH.INDEX_EXACT,
+      idPlan,
+      reason: "all criteria are satisfied by field indexes",
+    };
+  }
+
+  if (idPlan) {
+    return {
+      path: READ_PATH.INDEX_PARTIAL,
+      idPlan,
+      reason: "field indexes narrow part of the criteria",
+    };
+  }
+
+  return {
+    path: READ_PATH.SCAN,
+    idPlan: allIdsPlan(),
+    reason: "criteria are not indexable",
+  };
+}
+
+export function planCount(criteria = {}, options = {}, adapter) {
+  const read = planRead(criteria, options, adapter);
+  if (read.path === READ_PATH.ORDERED_INDEX) {
+    if (!hasObjectCriteria(criteria)) return { ...read, path: READ_PATH.SCAN, idPlan: allIdsPlan() };
+    const idPlan = planIndexedIds(criteria, adapter);
+    return idPlan
+      ? { ...read, path: idPlan.exact ? READ_PATH.INDEX_EXACT : READ_PATH.INDEX_PARTIAL, idPlan }
+      : { ...read, path: READ_PATH.SCAN, idPlan: allIdsPlan() };
+  }
+  return read;
+}
+
+export function planAggregate(criteria = {}, spec = {}, options = {}, adapter) {
+  const normalized = normalizeAggregateSpec(spec);
+  const hasCriteria = hasObjectCriteria(criteria);
+  const includeId = typeof options.includeId === "function" ? options.includeId : null;
+  const includeIds = options.includeIds instanceof Set ? options.includeIds : null;
+  const idPlan = aggregateIdPlan(criteria, includeId, includeIds, adapter);
+  const metricColumns = readyColumns(metricColumnPaths(normalized), adapter);
+
+  if (!metricColumns.ready) {
+    return aggregateScanPlan(normalized, `missing or building column index for ${metricColumns.missing}`);
+  }
+
+  if (normalized.groupBy.length === 0) {
+    if (!hasCriteria && !includeId && !includeIds && canUseColumnSummary(normalized)) {
+      return {
+        path: AGGREGATE_PATH.COLUMN_SUMMARY,
+        normalized,
+        columns: metricColumns.columns,
+        reason: "flat aggregate can be answered from column summaries",
+      };
+    }
+
+    if (!idPlan) return aggregateScanPlan(normalized, "criteria cannot be narrowed by indexes");
+    return {
+      path: AGGREGATE_PATH.COLUMN_SCAN,
+      normalized,
+      columns: metricColumns.columns,
+      idPlan,
+      reason: "flat aggregate can scan column values over planned ids",
+    };
+  }
+
+  if (!idPlan) return aggregateScanPlan(normalized, "criteria cannot be narrowed by indexes");
+
+  const scopeShards = Array.isArray(options.scopeShards) ? options.scopeShards : null;
+  if (normalized.groupBy.length === 1 && !isRowCountOnlyAggregate(normalized) && idPlan.exact && idPlan.ids instanceof Set &&
+      adapter.hasGroupForwardIndex?.(normalized.groupBy[0])) {
+    return {
+      path: AGGREGATE_PATH.GROUP_FORWARD_INDEX,
+      normalized,
+      columns: metricColumns.columns,
+      groupBy: normalized.groupBy[0],
+      groupIndex: adapter.fieldIndex(normalized.groupBy[0]),
+      idPlan,
+      reason: "groupBy field can be read from the field index forward map and metrics have column indexes",
+    };
+  }
+
+  if (normalized.groupBy.length === 1 && adapter.hasGroupIndex(normalized.groupBy[0])) {
+    return {
+      path: AGGREGATE_PATH.GROUP_INDEX,
+      normalized,
+      columns: metricColumns.columns,
+      groupBy: normalized.groupBy[0],
+      groupIndex: adapter.fieldIndex(normalized.groupBy[0]),
+      idPlan,
+      reason: "groupBy field has a field index and metrics have column indexes",
+    };
+  }
+
+  const groupColumns = readyColumns(normalized.groupBy, adapter);
+  if (!groupColumns.ready) {
+    return aggregateScanPlan(normalized, `missing or building group column index for ${groupColumns.missing}`);
+  }
+
+  return {
+    path: AGGREGATE_PATH.GROUP_COLUMN_SCAN,
+    normalized,
+    columns: mergeColumnMaps(metricColumns.columns, groupColumns.columns),
+    idPlan,
+    reason: "group fields and metric fields have ready column indexes",
+  };
+}
+
+function planIndexedIds(criteria, adapter) {
+  const exact = adapter.idsForCriteria(criteria);
+  if (exact !== null) return { ids: exact, exact: true, verify: false };
+
+  const partial = adapter.partialIdsForCriteria(criteria);
+  if (partial !== null) return { ids: partial, exact: false, verify: true };
+
+  return null;
+}
+
+function aggregateIdPlan(criteria, includeId, includeIds, adapter) {
+  if (!hasObjectCriteria(criteria)) {
+    if (includeIds && includeId) return { ids: adapter.filterIds(includeIds, includeId), exact: true, verify: false };
+    if (includeIds) return { ids: includeIds, exact: true, verify: false };
+    if (includeId) return { ids: adapter.idsForAll(includeId), exact: true, verify: false };
+    return allIdsPlan();
+  }
+
+  const indexed = planIndexedIds(criteria, adapter);
+  if (!indexed) return null;
+  if (includeIds) return { ...indexed, scopeIds: includeIds };
+  if (!includeId) return indexed;
+  return {
+    ...indexed,
+    ids: adapter.filterIds(indexed.ids, includeId),
+  };
+}
+
+function allIdsPlan() {
+  return { ids: null, exact: true, verify: false };
+}
+
+function aggregateScanPlan(normalized, reason) {
+  return {
+    path: AGGREGATE_PATH.SCAN,
+    normalized,
+    reason,
+  };
+}
+
+function readyColumns(paths, adapter) {
+  const columns = new Map();
+  for (const path of unique(paths)) {
+    const column = adapter.column(path);
+    if (!column || !column.ready) return { ready: false, missing: path, columns };
+    columns.set(path, column);
+  }
+  return { ready: true, columns };
+}
+
+function metricColumnPaths(normalized) {
+  const paths = [];
+  for (const field of normalized.fields) {
+    if (field.operator === "$count" && field.operand === true) continue;
+    paths.push(field.operand);
+  }
+  return paths;
+}
+
+function canUseColumnSummary(normalized) {
+  return normalized.fields.every((field) => {
+    return field.operator === "$sum" || field.operator === "$avg" || field.operator === "$count";
+  });
+}
+
+function isRowCountOnlyAggregate(normalized) {
+  return normalized.fields.length > 0 &&
+    normalized.fields.every((field) => field.operator === "$count" && field.operand === true);
+}
+
+function mergeColumnMaps(left, right) {
+  const merged = new Map(left);
+  for (const [path, column] of right) merged.set(path, column);
+  return merged;
+}
+
+function hasObjectCriteria(criteria) {
+  return Boolean(criteria && typeof criteria === "object" && !Array.isArray(criteria) && Object.keys(criteria).length > 0);
+}
+
+function unique(values) {
+  return [...new Set(values.map((value) => String(value)).filter(Boolean))];
+}

package/src/query.js

@@ -0,0 +1,416 @@
+const rhsOperators = new Set([
+  "$cb",
+  "$deepEquals",
+  "$eq",
+  "$equal",
+  "$ne",
+  "$not",
+  "$nor",
+  "$and",
+  "$or",
+  "$any",
+  "$all",
+  "$null",
+  "$in",
+  "$likeI",
+  "$like",
+  "$startsWith",
+  "$endsWith",
+  "$elemMatch",
+  "$contains",
+  "$nin",
+  "$regex",
+  "$gte",
+  "$gt",
+  "$lt",
+  "$lte",
+  "$before",
+  "$after",
+  "$type",
+  "$size",
+  "$mod",
+  "$between",
+  "$exists",
+]);
+
+const queryControls = new Set([
+  "$aggregate",
+  "$cursor",
+  "$limit",
+  "$offset",
+  "$page",
+  "$pageSize",
+  "$refDepth",
+  "$skip",
+  "$sort",
+  "$vector",
+]);
+
+export function getPathValue(source, path) {
+  if (source === null || source === undefined) return undefined;
+  const key = String(path);
+  if (key === "_id") return source._id;
+  if (key === "id") return source.id;
+
+  return key.split(".").reduce((cursor, part) => {
+    if (cursor === null || cursor === undefined || typeof cursor !== "object") return undefined;
+    return cursor[part];
+  }, source);
+}
+
+export function stableKey(value) {
+  return JSON.stringify(sortForKey(value));
+}
+
+export function matchesQuery(object, query = {}) {
+  const criteria = stripQueryControls(query);
+  if (!isPlainObject(criteria)) return false;
+
+  for (const [path, expected] of Object.entries(criteria)) {
+    if (path === "$and") {
+      if (!logicalConstraints(expected).every((constraint) => matchesQuery(object, constraint))) return false;
+      continue;
+    }
+    if (path === "$or") {
+      if (!logicalConstraints(expected).some((constraint) => matchesQuery(object, constraint))) return false;
+      continue;
+    }
+    if (path === "$nor") {
+      if (logicalConstraints(expected).some((constraint) => matchesQuery(object, constraint))) return false;
+      continue;
+    }
+    if (path === "$not") {
+      if (matchesQuery(object, expected)) return false;
+      continue;
+    }
+    if (path === "$count") {
+      if (!matchesCount(object, expected)) return false;
+      continue;
+    }
+    if (path === "$where") {
+      if (!matchesWhere(object, expected)) return false;
+      continue;
+    }
+
+    const actual = getPathValue(object, path);
+    if (!matchesConstraint(actual, expected, path)) return false;
+  }
+
+  return true;
+}
+
+export function stripQueryControls(query = {}) {
+  if (!isPlainObject(query)) return query;
+  const out = {};
+  for (const [key, value] of Object.entries(query)) {
+    if (!queryControls.has(key)) out[key] = value;
+  }
+  return out;
+}
+
+export function splitQueryControls(query = {}) {
+  if (!isPlainObject(query)) {
+    return {
+      criteria: {},
+      controls: {},
+    };
+  }
+  const criteria = {};
+  const controls = {};
+  for (const [key, value] of Object.entries(query)) {
+    if (queryControls.has(key)) controls[key] = value;
+    else criteria[key] = value;
+  }
+  return { criteria, controls };
+}
+
+export function parseQuerySort(value) {
+  if (typeof value === "string" && value.length > 0) {
+    return value.startsWith("-")
+      ? { sortBy: value.slice(1), sort: "desc" }
+      : { sortBy: value.replace(/^\+/, ""), sort: "asc" };
+  }
+  if (isPlainObject(value)) {
+    const [field, direction] = Object.entries(value)[0] || [];
+    if (!field) return {};
+    return { sortBy: field, sort: direction === -1 || direction === "desc" ? "desc" : "asc" };
+  }
+  return {};
+}
+
+export function queryInteger(value) {
+  if (value === null || value === undefined || value === "") return undefined;
+  const number = Number(value);
+  return Number.isFinite(number) && number >= 0 ? Math.floor(number) : undefined;
+}
+
+export function firstQueryInteger(...values) {
+  for (const value of values) {
+    const parsed = queryInteger(value);
+    if (parsed !== undefined) return parsed;
+  }
+  return undefined;
+}
+
+export function queryPageOffset(page, pageSize) {
+  const parsedPage = queryInteger(page);
+  return parsedPage === undefined || pageSize === undefined
+    ? undefined
+    : Math.max(0, parsedPage - 1) * pageSize;
+}
+
+export function compareSortValues(left, right) {
+  if (Object.is(left, right)) return 0;
+  if (left === undefined || left === null) return 1;
+  if (right === undefined || right === null) return -1;
+  if (typeof left === "number" && typeof right === "number") return left - right;
+  return String(left).localeCompare(String(right));
+}
+
+export function matchesConstraint(actual, expected, parentKey) {
+  if (expected === actual) return true;
+  if (expected instanceof RegExp) return regexMatches(actual, expected);
+  if (Array.isArray(expected)) return inOperator(actual, expected);
+
+  if (expected && typeof expected === "object") {
+    if (expected instanceof Date) return equal(actual, expected.getTime());
+    if (expected.$regex !== undefined) return regexMatches(actual, regexFromLegacySpec(expected));
+
+    for (const [operator, operand] of Object.entries(expected)) {
+      if (!rhsOperators.has(operator)) return equal(actual, expected);
+      if (!matchesOperator(operator, actual, operand, parentKey)) return false;
+    }
+    return true;
+  }
+
+  if (Array.isArray(actual)) return actual.some((entry) => equal(entry, expected));
+  if (isNullishConstraint(expected)) return nullOperator(actual);
+  return equal(actual, expected);
+}
+
+function matchesOperator(operator, actual, operand, parentKey) {
+  switch (operator) {
+    case "$cb":
+      return typeof operand === "function" && Boolean(operand(actual, parentKey));
+    case "$deepEquals":
+      return stableKey(actual) === stableKey(operand);
+    case "$eq":
+    case "$equal":
+      return equal(actual, operand);
+    case "$ne":
+      return !matchesConstraint(actual, operand, parentKey);
+    case "$not":
+      return !matchesConstraint(actual, operand, parentKey);
+    case "$nor":
+      return !orOperator(actual, operand, parentKey);
+    case "$and":
+    case "$all":
+      return andOperator(actual, operand, parentKey);
+    case "$or":
+    case "$any":
+      return orOperator(actual, operand, parentKey);
+    case "$null":
+      return Boolean(operand) ? nullOperator(actual) : !nullOperator(actual);
+    case "$in":
+      return inOperator(actual, operand);
+    case "$likeI":
+      return likeOperator(actual, operand, { ignoreCase: true });
+    case "$like":
+      return likeOperator(actual, operand);
+    case "$startsWith":
+      return stringList(actual).some((value) => value.startsWith(String(operand)));
+    case "$endsWith":
+      return stringList(actual).some((value) => value.endsWith(String(operand)));
+    case "$elemMatch":
+      return Array.isArray(actual) && actual.some((entry) => matchesQuery(entry, operand));
+    case "$contains":
+      return contains(actual, operand);
+    case "$nin":
+      return !inOperator(actual, operand);
+    case "$regex":
+      return regexMatches(actual, regexFromOperand(operand));
+    case "$gte":
+      return !nullOperator(actual) && actual >= resolveOperand(operand);
+    case "$gt":
+      return !nullOperator(actual) && actual > resolveOperand(operand);
+    case "$lt":
+      return !nullOperator(actual) && actual < resolveOperand(operand);
+    case "$lte":
+      return !nullOperator(actual) && actual <= resolveOperand(operand);
+    case "$before":
+      return beforeAfter(actual, operand, "$before");
+    case "$after":
+      return beforeAfter(actual, operand, "$after");
+    case "$type":
+      return typeOf(actual) === operand;
+    case "$size":
+      return sizeOf(actual) === Number(operand);
+    case "$mod":
+      return Array.isArray(operand) && operand.length >= 2 && actual % operand[0] == operand[1];
+    case "$between":
+      return Array.isArray(operand) &&
+        operand.length >= 2 &&
+        matchesConstraint(actual, { $gt: operand[0], $lt: operand[1] }, parentKey);
+    case "$exists":
+      return existsOperator(actual, operand);
+    default:
+      return false;
+  }
+}
+
+function matchesCount(object, condition) {
+  if (!isPlainObject(condition) || !Array.isArray(condition.$constraints)) return false;
+  const count = condition.$constraints.filter((constraint) => matchesQuery(object, constraint)).length;
+  return matchesConstraint(count, condition.$constraint);
+}
+
+function matchesWhere(object, condition) {
+  if (typeof condition !== "function") return false;
+  return Boolean(condition.call(object, object));
+}
+
+function logicalConstraints(value) {
+  if (Array.isArray(value)) return value;
+  if (!isPlainObject(value)) return [];
+  return Object.entries(value).map(([key, constraint]) => ({ [key]: constraint }));
+}
+
+function andOperator(actual, constraints, parentKey) {
+  if (!Array.isArray(constraints)) return false;
+  return constraints.every((constraint) => matchesConstraint(actual, constraint, parentKey));
+}
+
+function orOperator(actual, constraints, parentKey) {
+  if (!Array.isArray(constraints)) return false;
+  const values = Array.isArray(actual) ? actual : [actual];
+  return values.some((value) => constraints.some((constraint) => matchesConstraint(value, constraint, parentKey)));
+}
+
+function inOperator(actual, expected) {
+  if (!Array.isArray(expected)) return false;
+  const values = Array.isArray(actual) ? actual : [actual];
+  return values.some((value) => expected.some((constraint) => matchesConstraint(value, constraint)));
+}
+
+function contains(actual, expected) {
+  if (typeof actual === "string") return actual.includes(String(expected));
+  if (Array.isArray(actual)) return actual.some((entry) => equal(entry, expected));
+  return false;
+}
+
+function regexMatches(actual, regex) {
+  if (!(regex instanceof RegExp)) return false;
+  const values = Array.isArray(actual) ? actual : [actual];
+  return values.some((value) => {
+    regex.lastIndex = 0;
+    return regex.test(value);
+  });
+}
+
+function regexFromLegacySpec(spec) {
+  if (spec.$regex && typeof spec.$regex === "object" && typeof spec.$regex.pattern === "string") {
+    return new RegExp(spec.$regex.pattern, spec.$regex.flags || spec.$options || "");
+  }
+  return new RegExp(spec.$regex, spec.$options || "");
+}
+
+function regexFromOperand(operand) {
+  if (operand instanceof RegExp) return operand;
+  if (typeof operand === "string") return new RegExp(operand);
+  if (operand && typeof operand === "object" && typeof operand.pattern === "string") {
+    return new RegExp(operand.pattern, operand.flags || "");
+  }
+  return null;
+}
+
+function likeOperator(actual, expected, { ignoreCase = false } = {}) {
+  const needle = ignoreCase ? String(expected).toLowerCase() : String(expected);
+  return stringList(actual).some((value) => {
+    const haystack = ignoreCase ? value.toLowerCase() : value;
+    return haystack.includes(needle);
+  });
+}
+
+function stringList(value) {
+  const values = Array.isArray(value) ? value : [value];
+  return values.filter((entry) => typeof entry === "string");
+}
+
+function beforeAfter(actual, operand, operator) {
+  const left = dateComparable(actual);
+  const right = dateComparable(resolveOperand(operand));
+  if (!Number.isFinite(left) || !Number.isFinite(right)) return false;
+  return operator === "$before" ? left <= right : left >= right;
+}
+
+function dateComparable(value) {
+  if (typeof value === "string") return Date.parse(value);
+  if (value instanceof Date) return value.getTime();
+  return value;
+}
+
+function resolveOperand(value) {
+  if (value && typeof value === "object" && value.$date !== undefined) return Date.parse(value.$date);
+  return value;
+}
+
+function typeOf(value) {
+  if (Array.isArray(value)) return "array";
+  return typeof value;
+}
+
+function sizeOf(value) {
+  if (value === null || value === undefined) return NaN;
+  if (typeof value === "string" || Array.isArray(value)) return value.length;
+  if (typeof value === "object") return Object.keys(value).length;
+  return NaN;
+}
+
+function existsOperator(value, expected) {
+  return (value !== undefined && value !== null) === Boolean(expected);
+}
+
+function nullOperator(value) {
+  if (value === "" || value === null || value === undefined) return true;
+  if (Array.isArray(value)) return value.every((entry) => nullOperator(entry));
+  return false;
+}
+
+function isNullishConstraint(value) {
+  return value === "" || value === null || value === undefined;
+}
+
+function equal(left, right) {
+  if (Object.is(left, right)) return true;
+  if (Array.isArray(left)) return left.some((entry) => equal(entry, right));
+  if (isNullishConstraint(right)) return nullOperator(left);
+  if (isNullishConstraint(left)) return false;
+
+  if (left instanceof Date) {
+    if (right instanceof Date) return left.getTime() == right.getTime();
+    if (typeof right === "number") return left.getTime() == right;
+    if (typeof right === "string") return left.getTime() == Date.parse(right);
+  }
+
+  if ((left && typeof left === "object") || (right && typeof right === "object")) {
+    return stableKey(left) === stableKey(right);
+  }
+
+  return left == right;
+}
+
+function sortForKey(value) {
+  if (Array.isArray(value)) return value.map(sortForKey);
+  if (value && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([key, entry]) => [key, sortForKey(entry)]),
+    );
+  }
+  return value;
+}
+
+function isPlainObject(value) {
+  return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}