@x12i/static-memorix-explorer-api 1.0.0

package/dist/engine/narratives.js

@@ -0,0 +1,77 @@
+import path from "node:path";
+import { store } from "../storage/InMemoryStore.js";
+import { CONTENT_TYPES } from "../types.js";
+import { listJsonFiles, basenameNoExt, readJson } from "../storage/fs.js";
+import { METADATA_DIR } from "../config.js";
+import { isDir } from "../storage/fs.js";
+import fs from "node:fs";
+/**
+ * Discover narrative keys tagged on records via doc.narratives.*.
+ */
+export function discoverRecordNarrativeTags(entityName) {
+    const tags = new Set();
+    for (const ct of CONTENT_TYPES) {
+        const records = store.getCollection(entityName, ct);
+        for (const doc of records) {
+            const narratives = doc.narratives || {};
+            for (const key of Object.keys(narratives))
+                tags.add(key);
+        }
+    }
+    return [...tags];
+}
+export function mergeCatalogWithSignals(authored, recordTags) {
+    const result = [];
+    const authoredKeys = Object.keys(authored || {});
+    for (const key of authoredKeys) {
+        result.push({ ...authored[key], key, signaled: recordTags.includes(key) });
+    }
+    for (const tag of recordTags) {
+        if (!authoredKeys.includes(tag)) {
+            result.push({ key: tag, signaled: true });
+        }
+    }
+    return result;
+}
+export function getMergedNarratives(entityName) {
+    if (entityName) {
+        const authored = store.getMeta(`narratives/${entityName}`) || {};
+        const recordTags = discoverRecordNarrativeTags(entityName);
+        return mergeCatalogWithSignals(authored, recordTags);
+    }
+    // global across all entities
+    const dir = path.join(METADATA_DIR, "narratives");
+    const out = {};
+    for (const file of listJsonFiles(dir)) {
+        const name = basenameNoExt(file);
+        const authored = readJson(file, {});
+        const recordTags = discoverRecordNarrativeTags(name);
+        out[name] = mergeCatalogWithSignals(authored, recordTags);
+    }
+    return out;
+}
+export function getRawNarratives(entityName) {
+    return (store.getMeta(`narratives/${entityName}`) || {});
+}
+export function queryNarrativeRecords(entityName, key) {
+    const results = [];
+    for (const ct of CONTENT_TYPES) {
+        const records = store.getCollection(entityName, ct);
+        for (const doc of records) {
+            const narratives = doc.narratives || {};
+            const val = narratives[key];
+            if (val === true || val === key)
+                results.push(doc);
+        }
+    }
+    return results;
+}
+export function listNarrativeEntities() {
+    const dir = path.join(METADATA_DIR, "narratives");
+    if (!isDir(dir))
+        return [];
+    return fs
+        .readdirSync(dir)
+        .filter((f) => f.endsWith(".json"))
+        .map((f) => basenameNoExt(f));
+}

package/dist/engine/objectTypes.js

@@ -0,0 +1,71 @@
+import { store } from "../storage/InMemoryStore.js";
+import { CONTENT_TYPES } from "../types.js";
+/**
+ * Full-scan all record arrays in ./data/{name}/*.json, compute property
+ * occurrence percentages, data types, and paths.
+ */
+export function computeRootPropertyCatalog(objectType) {
+    const counts = new Map();
+    const typesMap = new Map();
+    const samples = new Map();
+    let totalDocs = 0;
+    for (const ct of CONTENT_TYPES) {
+        const records = store.getCollection(objectType, ct);
+        for (const doc of records) {
+            totalDocs++;
+            const flat = flatten(doc);
+            for (const [p, v] of Object.entries(flat)) {
+                counts.set(p, (counts.get(p) || 0) + 1);
+                const t = typeOf(v);
+                if (!typesMap.has(p))
+                    typesMap.set(p, new Set());
+                typesMap.get(p).add(t);
+                if (!samples.has(p))
+                    samples.set(p, []);
+                const arr = samples.get(p);
+                if (arr.length < 3 && !arr.includes(v))
+                    arr.push(v);
+            }
+        }
+    }
+    const entries = [];
+    for (const [p, c] of counts.entries()) {
+        entries.push({
+            path: p,
+            occurrencePct: totalDocs ? Math.round((c / totalDocs) * 1000) / 10 : 0,
+            types: [...(typesMap.get(p) || [])],
+            sampleValues: samples.get(p) || [],
+        });
+    }
+    entries.sort((a, b) => b.occurrencePct - a.occurrencePct);
+    // Persist back to metadata
+    const meta = store.getMeta(`object-types/${objectType}`) || { name: objectType };
+    meta.rootPropertyCatalog = entries;
+    store.setMeta(`object-types/${objectType}`, meta);
+    return entries;
+}
+function flatten(obj, prefix = "") {
+    const out = {};
+    if (obj == null || typeof obj !== "object") {
+        if (prefix)
+            out[prefix] = obj;
+        return out;
+    }
+    for (const [k, v] of Object.entries(obj)) {
+        const key = prefix ? `${prefix}.${k}` : k;
+        if (v != null && typeof v === "object" && !Array.isArray(v)) {
+            Object.assign(out, flatten(v, key));
+        }
+        else {
+            out[key] = v;
+        }
+    }
+    return out;
+}
+function typeOf(v) {
+    if (v === null)
+        return "null";
+    if (Array.isArray(v))
+        return "array";
+    return typeof v;
+}

package/dist/engine/query.js

@@ -0,0 +1,191 @@
+import mingo from "mingo";
+export function parseFilterToken(token) {
+    const idx1 = token.indexOf(":");
+    if (idx1 === -1)
+        return null;
+    const field = token.slice(0, idx1);
+    const rest = token.slice(idx1 + 1);
+    const idx2 = rest.indexOf(":");
+    if (idx2 === -1) {
+        // treat as eq
+        return { field, op: "eq", value: coerce(rest), raw: token };
+    }
+    const op = rest.slice(0, idx2);
+    const value = rest.slice(idx2 + 1);
+    // Narrative virtual filter interception: narrativeId:eq:has-vulns /
+    // narrativeKey:eq:has-vulns  ->  evaluate doc.narratives['has-vulns'].
+    if (field === "narrativeId" || field === "narrativeKey") {
+        return {
+            field,
+            op: op || "eq",
+            value: coerceForOp(op || "eq", value),
+            raw: token,
+            isVirtual: true,
+        };
+    }
+    if (!["eq", "ne", "gt", "gte", "lt", "lte", "in", "nin", "exists", "regex"].includes(op)) {
+        return { field, op: "eq", value: coerce(rest), raw: token };
+    }
+    return { field, op, value: coerceForOp(op, value), raw: token };
+}
+export function coerce(value) {
+    if (value === "true")
+        return true;
+    if (value === "false")
+        return false;
+    if (value === "null")
+        return null;
+    if (value !== "" && !isNaN(Number(value)))
+        return Number(value);
+    return value;
+}
+function coerceForOp(op, value) {
+    if (op === "in" || op === "nin") {
+        return value
+            .split("|")
+            .map((v) => coerce(v.trim()))
+            .filter((v) => v !== "");
+    }
+    if (op === "exists") {
+        return !(value === "false" || value === "0");
+    }
+    if (op === "regex") {
+        return value;
+    }
+    return coerce(value);
+}
+/**
+ * Parse the filter param. Supports repeated values (array) and `;` separators.
+ * Returns virtual narrative filters separately so the caller can rewrite them.
+ */
+export function parseExplorerFilters(filter) {
+    const tokens = toArray(filter)
+        .flatMap((f) => f.split(";"))
+        .map((f) => f.trim())
+        .filter(Boolean);
+    const criteria = {};
+    const virtual = [];
+    for (const token of tokens) {
+        const parsed = parseFilterToken(token);
+        if (!parsed)
+            continue;
+        if (parsed.isVirtual) {
+            virtual.push(parsed);
+            continue;
+        }
+        applyFilterToCriteria(criteria, parsed);
+    }
+    return { criteria, virtual };
+}
+function applyFilterToCriteria(criteria, parsed) {
+    const { field, op, value } = parsed;
+    switch (op) {
+        case "eq":
+            criteria[field] = value;
+            break;
+        case "ne":
+            criteria[field] = { $ne: value };
+            break;
+        case "gt":
+            criteria[field] = { $gt: value };
+            break;
+        case "gte":
+            criteria[field] = { $gte: value };
+            break;
+        case "lt":
+            criteria[field] = { $lt: value };
+            break;
+        case "lte":
+            criteria[field] = { $lte: value };
+            break;
+        case "in":
+            criteria[field] = { $in: value };
+            break;
+        case "nin":
+            criteria[field] = { $nin: value };
+            break;
+        case "exists":
+            criteria[field] = { $exists: value };
+            break;
+        case "regex":
+            criteria[field] = { $regex: value, $options: "i" };
+            break;
+    }
+}
+export function parseSortParams(sort) {
+    const tokens = toArray(sort)
+        .flatMap((s) => s.split(";"))
+        .map((s) => s.trim())
+        .filter(Boolean);
+    const sortSpec = {};
+    for (const token of tokens) {
+        const [field, dirRaw] = token.split(":");
+        const dir = (dirRaw || "asc").toLowerCase();
+        sortSpec[field] = dir === "desc" ? -1 : 1;
+    }
+    return sortSpec;
+}
+function toArray(v) {
+    if (v === undefined || v === null)
+        return [];
+    return Array.isArray(v) ? v : [v];
+}
+/**
+ * Resolve virtual narrative filters into a predicate that checks
+ * doc.narratives[key] presence / truthiness.
+ */
+export function applyVirtualNarrativeFilters(records, virtual) {
+    if (virtual.length === 0)
+        return records;
+    return records.filter((doc) => {
+        const narratives = doc.narratives || {};
+        return virtual.every((vf) => {
+            const key = String(vf.value);
+            const actual = narratives[key];
+            if (vf.op === "exists")
+                return vf.value ? actual !== undefined : actual === undefined;
+            if (vf.op === "eq")
+                return actual === true || actual === vf.value;
+            return true;
+        });
+    });
+}
+/**
+ * Execute a query against an in-memory record array using mingo, with
+ * full-text search, virtual narrative filters, sorting, and pagination.
+ */
+export function queryMockCollection(records, queryParams) {
+    let pipeline = [...records];
+    // 1. Intercept narrative virtual filters
+    const { criteria, virtual } = parseExplorerFilters(queryParams.filter);
+    pipeline = applyVirtualNarrativeFilters(pipeline, virtual);
+    // 2. Full-Text Search
+    const term = (queryParams.searchText || queryParams.q || "").toLowerCase();
+    if (term) {
+        pipeline = pipeline.filter((doc) => JSON.stringify(doc).toLowerCase().includes(term));
+    }
+    // 3. Mingo Mongo Query Execution
+    const query = new mingo.Query(criteria);
+    let cursor = query.find(pipeline);
+    // 4. Sorting
+    const sortSpec = parseSortParams(queryParams.sort);
+    if (Object.keys(sortSpec).length > 0) {
+        cursor = cursor.sort(sortSpec);
+    }
+    // Materialize all matched docs (mingo cursors are single-use, so we cannot
+    // call .count() and then .all() on the same cursor).
+    const matched = cursor.all();
+    const total = matched.length;
+    // 5. Pagination
+    const limit = Math.min(Number(queryParams.limit) || 50, 500);
+    const offset = Number(queryParams.offset) || Number(queryParams.skip) || 0;
+    const rows = matched.slice(offset, offset + limit);
+    return {
+        rows,
+        page: {
+            offset,
+            limit,
+            total: queryParams.includeTotal ? total : undefined,
+        },
+    };
+}

package/dist/engine/records.js

@@ -0,0 +1,152 @@
+import { store } from "../storage/InMemoryStore.js";
+import { CONTENT_TYPES, } from "../types.js";
+import { queryMockCollection } from "../engine/query.js";
+import { executeListDescriptor, getListDescriptor } from "../engine/lists.js";
+import { findByIdentity } from "../engine/identity.js";
+export function inferIdField(entityName, ct) {
+    if (ct === "memory")
+        return "memoryId";
+    const sample = store.getCollection(entityName, ct)[0];
+    if (sample) {
+        if ("recordId" in sample)
+            return "recordId";
+        if ("entityId" in sample)
+            return "entityId";
+        if ("id" in sample)
+            return "id";
+    }
+    return "recordId";
+}
+/**
+ * Find a record within a collection using an identity map (recordId, entityId,
+ * eventId, knowledgeId, or memoryId). At least one identity key is required.
+ */
+export function findRecordByIdentity(entityName, ct, identity) {
+    const records = store.getCollection(entityName, ct);
+    return findByIdentity(records, identity);
+}
+export function findRecord(entityName, ct, recordId) {
+    const idField = inferIdField(entityName, ct);
+    return store
+        .getCollection(entityName, ct)
+        .find((d) => String(d[idField]) === String(recordId));
+}
+export async function getRecordsCollection(params) {
+    const target = params.target;
+    // The `memory` target routes to the memory tier regardless of entityName.
+    if (target === "memory") {
+        const ct = "memory";
+        const records = store.getCollection("memory", ct);
+        return queryMockCollection(records, params);
+    }
+    const entityName = params.entityName;
+    const mode = params.mode || "auto";
+    const listId = params.listId;
+    if (mode === "descriptor" || (mode === "auto" && listId)) {
+        const resolvedListId = listId || `${entityName}-default`;
+        const listSpec = getListDescriptor(resolvedListId);
+        if (listSpec) {
+            return executeListDescriptor(listSpec, params);
+        }
+        // fall through to raw if descriptor missing
+    }
+    const ct = params.contentType || "snapshots";
+    const records = store.getCollection(entityName, ct);
+    return queryMockCollection(records, params);
+}
+export function getRecordsFull(entityName, recordId) {
+    const identity = { recordId };
+    const snapshot = findRecordByIdentity(entityName, "snapshots", identity);
+    const analysis = findRecordByIdentity(entityName, "analysis", identity);
+    const decisions = findRecordByIdentity(entityName, "decisions", identity);
+    const memory = findRecordByIdentity(entityName, "memory", identity);
+    return {
+        recordId,
+        entityName,
+        contentTypes: { snapshot, analysis, decisions, memory },
+    };
+}
+export function getRecordsItem(entityName, recordId, itemDescriptor) {
+    const record = findRecordByIdentity(entityName, "snapshots", { recordId });
+    if (!record)
+        return undefined;
+    if (!itemDescriptor)
+        return record;
+    // Hydrate sections/fields from metadata/items
+    const sections = itemDescriptor.sections || [];
+    const out = { recordId, entityName };
+    for (const section of sections) {
+        const fields = section.fields || [];
+        out[section.key || section.id] = fields.map((f) => ({
+            field: f.field || f.key,
+            value: getPath(record, f.field || f.key),
+            label: f.label,
+        }));
+    }
+    return out;
+}
+/**
+ * Fetch a single content record by id. Supports the memory tier via memoryId.
+ * If entityName is empty and contentType is "memory", defaults to "memory".
+ * Otherwise, searches across all object types in the store.
+ */
+export function getRecordContent(entityName, contentType, identity) {
+    if (contentType === "memory") {
+        entityName = entityName || "memory";
+        return findRecordByIdentity(entityName, contentType, identity);
+    }
+    if (!entityName) {
+        // Search across all collections for the identity key
+        const keys = store.getCollectionKeys();
+        for (const key of keys) {
+            const [ot, ct] = key.split("/");
+            if (ct !== contentType)
+                continue;
+            const found = findRecordByIdentity(ot, contentType, identity);
+            if (found)
+                return found;
+        }
+        return undefined;
+    }
+    return findRecordByIdentity(entityName, contentType, identity);
+}
+export function getRawCollection(entityName, collectionName) {
+    // memory tier is keyed by a standalone `memory` object type
+    if (collectionName === "memory" || entityName === "memory") {
+        return store.getCollection("memory", "memory");
+    }
+    const ct = collectionName;
+    if (CONTENT_TYPES.includes(ct)) {
+        return store.getCollection(entityName, ct);
+    }
+    // allow objectType/contentType form
+    const [ot, c] = collectionName.split("/");
+    if (CONTENT_TYPES.includes(c)) {
+        return store.getCollection(ot, c);
+    }
+    return [];
+}
+export function getRawItem(collectionName, recordId) {
+    if (collectionName === "memory") {
+        return findRecordByIdentity("memory", "memory", { memoryId: recordId });
+    }
+    const ct = collectionName;
+    if (CONTENT_TYPES.includes(ct)) {
+        return findRecordByIdentity("memory", ct, { recordId }) ??
+            findRecord("__any__", ct, recordId);
+    }
+    const [entityName, c] = collectionName.split("/");
+    return findRecord(entityName, c, recordId);
+}
+export async function getWorkspaceRecords(params) {
+    // cross-entity query using a workspace list descriptor
+    const listId = params.listId || "workspace-default";
+    const listSpec = getListDescriptor(listId);
+    if (!listSpec) {
+        return { rows: [], page: { offset: 0, limit: 50 } };
+    }
+    return executeListDescriptor(listSpec, params);
+}
+function getPath(obj, path) {
+    return path.split(".").reduce((acc, k) => (acc == null ? undefined : acc[k]), obj);
+}

package/dist/engine/snapshots.js

@@ -0,0 +1,110 @@
+import { store } from "../storage/InMemoryStore.js";
+import { buildAssociatedBucket, aliasClientToStorage, resolveInclude, } from "../engine/aliases.js";
+import { inferIdField } from "../engine/records.js";
+export function getSnapshotRaw(objectType, recordId) {
+    const idField = inferIdField(objectType, "snapshots");
+    const rawDoc = store
+        .getCollection(objectType, "snapshots")
+        .find((d) => String(d[idField]) === String(recordId));
+    if (!rawDoc)
+        return undefined;
+    return { rawDoc, idField };
+}
+export function getSnapshotNormalized(objectType, recordId, include) {
+    const found = getSnapshotRaw(objectType, recordId);
+    if (!found)
+        return undefined;
+    const { rawDoc } = found;
+    const { associated, strippedDoc } = buildAssociatedBucket(rawDoc);
+    const flags = resolveInclude(include);
+    const out = { ...strippedDoc, associated };
+    if (flags) {
+        for (const flag of flags) {
+            applyIncludeFlag(out, rawDoc, associated, flag);
+        }
+    }
+    return out;
+}
+function applyIncludeFlag(out, rawDoc, associated, flag) {
+    switch (flag) {
+        case "data":
+            out.data = associated.data ?? rawDoc.associatedData ?? [];
+            break;
+        case "discovery":
+            out.discovery =
+                associated.discovery ?? rawDoc.associatedInferred ?? rawDoc.associatedDiscovery ?? [];
+            break;
+        case "analysis":
+            out.analysis = associated.analysis ?? rawDoc.associatedAnalysis ?? [];
+            break;
+        case "enrichment":
+            out.enrichment = rawDoc.enrichment ?? [];
+            break;
+        case "insights":
+            out.insights = rawDoc.insights ?? [];
+            break;
+        case "system":
+            out.system = rawDoc.system ?? {};
+            break;
+        case "associated":
+            out.associated = associated;
+            break;
+    }
+}
+export function getSnapshotAssociated(objectType, recordId) {
+    const found = getSnapshotRaw(objectType, recordId);
+    if (!found)
+        return undefined;
+    const { associated, properties } = buildAssociatedBucket(found.rawDoc);
+    return { associated, properties };
+}
+export function getSnapshotAssociatedProperty(objectType, recordId, propertyName) {
+    const found = getSnapshotRaw(objectType, recordId);
+    if (!found)
+        return undefined;
+    const { associated } = buildAssociatedBucket(found.rawDoc);
+    if (propertyName in associated) {
+        return associated[propertyName];
+    }
+    // try alias resolution
+    const { fallbacks } = aliasClientToStorage(propertyName);
+    for (const fb of fallbacks) {
+        if (fb in found.rawDoc)
+            return found.rawDoc[fb];
+    }
+    return undefined;
+}
+export function getAssociatedProperties(objectType) {
+    const records = store.getCollection(objectType, "snapshots");
+    const agg = new Map();
+    for (const doc of records) {
+        for (const k of Object.keys(doc)) {
+            if (k.startsWith("associated")) {
+                const existing = agg.get(k) || { source: k, count: 0 };
+                const v = doc[k];
+                existing.count += Array.isArray(v) ? v.length : v != null ? 1 : 0;
+                agg.set(k, existing);
+            }
+        }
+    }
+    return [...agg.entries()].map(([source, v]) => ({
+        propertyName: source,
+        source,
+        count: v.count,
+    }));
+}
+/**
+ * Deterministic fingerprint for association plan/apply/verify.
+ */
+export function computeAssociationFingerprint(objectType) {
+    const records = store.getCollection(objectType, "snapshots");
+    const serialized = JSON.stringify(records.map((r) => ({ id: r.id ?? r.recordId, keys: Object.keys(r) })));
+    return `fp:${objectType}:${hashString(serialized)}`;
+}
+export function hashString(s) {
+    let h = 5381;
+    for (let i = 0; i < s.length; i++) {
+        h = (h * 33) ^ s.charCodeAt(i);
+    }
+    return (h >>> 0).toString(16);
+}