@x12i/static-memorix 2.0.0

package/dist/storage/InMemoryStore.js

@@ -0,0 +1,242 @@
+import fs from "node:fs";
+import path from "node:path";
+import { CONTENT_TYPES } from "../types.js";
+import { MOCKS_DIR, METADATA_DIR, DATA_DIR, buildDataFilePrefix, buildMetadataFilePrefix, prefixedJsonFilename, validateRouteId, } from "../config.js";
+import { validateRecordArray } from "../engine/identity.js";
+import { isDir, listJsonFiles, readJson, writeJson, basenameNoExt, } from "./fs.js";
+import { DISK_FLUSH_DEBOUNCE_MS } from "../config.js";
+const DIRTY = Symbol("dirty");
+function routedMetadataPath(key) {
+    const parts = key.split("/");
+    if (parts.length === 1 && parts[0] === "agents") {
+        return path.join(METADATA_DIR, prefixedJsonFilename(buildMetadataFilePrefix(), "agents"));
+    }
+    if (parts.length !== 2) {
+        throw Object.assign(new Error(`Unsafe metadata key: ${key}`), { statusCode: 400 });
+    }
+    const [section, name] = parts;
+    if (!["object-types", "lists", "write-descriptors", "narratives"].includes(section)) {
+        throw Object.assign(new Error(`Unsupported metadata section: ${section}`), {
+            statusCode: 400,
+        });
+    }
+    validateRouteId(name, "metadata fixture name");
+    return path.join(METADATA_DIR, section, prefixedJsonFilename(buildMetadataFilePrefix(), name));
+}
+export class InMemoryStore {
+    collections = new Map(); // key: `${objectType}/${contentType}`
+    metadata = new Map();
+    flushTimers = new Map();
+    loaded = false;
+    /**
+     * Load everything from disk into memory. Idempotent.
+     */
+    async load() {
+        if (this.loaded)
+            return;
+        this.loadMetadataCatalog();
+        this.loadDataCollections();
+        this.loaded = true;
+    }
+    loadMetadataCatalog() {
+        const sections = {
+            "object-types": "object-types",
+            lists: "lists",
+            "write-descriptors": "write-descriptors",
+            narratives: "narratives",
+        };
+        const prefix = buildMetadataFilePrefix();
+        for (const [subdir] of Object.entries(sections)) {
+            const dir = path.join(METADATA_DIR, subdir);
+            const files = listJsonFiles(dir);
+            const logicalNames = new Set();
+            for (const file of files) {
+                const name = basenameNoExt(file);
+                if (!name.includes("--"))
+                    logicalNames.add(name);
+                if (name.startsWith(`${prefix}--`))
+                    logicalNames.add(name.slice(prefix.length + 2));
+            }
+            for (const name of logicalNames) {
+                const baseFile = path.join(dir, `${name}.json`);
+                const routedFile = path.join(dir, prefixedJsonFilename(prefix, name));
+                const source = fs.existsSync(routedFile) ? routedFile : baseFile;
+                const key = `${subdir}/${name}`;
+                this.metadata.set(key, { path: routedFile, data: readJson(source, {}) });
+            }
+        }
+        const baseAgentsFile = path.join(METADATA_DIR, "agents.json");
+        const routedAgentsFile = path.join(METADATA_DIR, prefixedJsonFilename(prefix, "agents"));
+        const agentsSource = fs.existsSync(routedAgentsFile) ? routedAgentsFile : baseAgentsFile;
+        if (fs.existsSync(agentsSource)) {
+            this.metadata.set("agents", {
+                path: routedAgentsFile,
+                data: readJson(agentsSource, { agents: [] }),
+            });
+        }
+    }
+    loadDataCollections() {
+        if (!isDir(DATA_DIR))
+            return;
+        for (const objectType of fs.readdirSync(DATA_DIR)) {
+            const otDir = path.join(DATA_DIR, objectType);
+            if (!isDir(otDir))
+                continue;
+            for (const ct of CONTENT_TYPES) {
+                const baseFile = path.join(otDir, `${ct}.json`);
+                const file = path.join(otDir, prefixedJsonFilename(buildDataFilePrefix(ct), ct));
+                const key = `${objectType}/${ct}`;
+                const source = fs.existsSync(file) ? file : baseFile;
+                const data = readJson(source, []);
+                // Startup validation: ensure records carry exactly one identity key
+                // (MemorixRecordArraySchema mock). Memory collections use memoryId.
+                const result = validateRecordArray(objectType, ct, data);
+                if (!result.ok) {
+                    for (const err of result.errors) {
+                        console.warn(`[static-memorix] schema warning: ${err}`);
+                    }
+                }
+                this.collections.set(key, {
+                    path: file,
+                    data,
+                });
+            }
+            // system overrides
+            const baseInvFile = path.join(DATA_DIR, "system", "inventory.json");
+            const invFile = path.join(DATA_DIR, "system", prefixedJsonFilename(buildDataFilePrefix("snapshots"), "inventory"));
+            const invSource = fs.existsSync(invFile) ? invFile : baseInvFile;
+            if (objectType === "system" && fs.existsSync(invSource)) {
+                this.collections.set("system/inventory", {
+                    path: invFile,
+                    data: readJson(invSource, []),
+                });
+            }
+        }
+    }
+    // ---- Metadata access ----
+    getMeta(key) {
+        return this.metadata.get(key)?.data;
+    }
+    setMeta(key, data, persist = true) {
+        const existing = this.metadata.get(key);
+        if (existing) {
+            existing.data = data;
+            existing[DIRTY] = true;
+        }
+        else {
+            this.metadata.set(key, {
+                path: routedMetadataPath(key),
+                data,
+                [DIRTY]: true,
+            });
+        }
+        if (persist)
+            this.scheduleMetaFlush(key);
+    }
+    /** Persist a routed tombstone so an unprefixed seed does not reappear. */
+    deleteMeta(key) {
+        const existing = this.metadata.get(key);
+        if (existing) {
+            existing.data = null;
+            existing[DIRTY] = true;
+        }
+        else {
+            this.metadata.set(key, {
+                path: routedMetadataPath(key),
+                data: null,
+                [DIRTY]: true,
+            });
+        }
+        this.scheduleMetaFlush(key);
+    }
+    getMetaKeys() {
+        return [...this.metadata.keys()];
+    }
+    // ---- Collection access ----
+    getCollection(objectType, ct) {
+        validateRouteId(objectType, "object type");
+        const key = `${objectType}/${ct}`;
+        const c = this.collections.get(key);
+        if (!c) {
+            const newC = {
+                path: path.join(DATA_DIR, objectType, prefixedJsonFilename(buildDataFilePrefix(ct), ct)),
+                data: [],
+            };
+            this.collections.set(key, newC);
+            return newC.data;
+        }
+        return c.data;
+    }
+    setCollection(objectType, ct, data) {
+        validateRouteId(objectType, "object type");
+        const key = `${objectType}/${ct}`;
+        const c = this.collections.get(key);
+        if (c) {
+            c.data = data;
+            c[DIRTY] = true;
+        }
+        else {
+            this.collections.set(key, {
+                path: path.join(DATA_DIR, objectType, prefixedJsonFilename(buildDataFilePrefix(ct), ct)),
+                data,
+                [DIRTY]: true,
+            });
+        }
+        this.scheduleCollectionFlush(key);
+    }
+    getCollectionKeys() {
+        return [...this.collections.keys()];
+    }
+    // ---- Flush machinery ----
+    scheduleCollectionFlush(key) {
+        const existing = this.flushTimers.get(key);
+        if (existing)
+            clearTimeout(existing);
+        const t = setTimeout(() => {
+            this.flushCollectionNow(key);
+            this.flushTimers.delete(key);
+        }, DISK_FLUSH_DEBOUNCE_MS);
+        this.flushTimers.set(key, t);
+    }
+    scheduleMetaFlush(key) {
+        const fkey = `meta:${key}`;
+        const existing = this.flushTimers.get(fkey);
+        if (existing)
+            clearTimeout(existing);
+        const t = setTimeout(() => {
+            this.flushMetaNow(key);
+            this.flushTimers.delete(fkey);
+        }, DISK_FLUSH_DEBOUNCE_MS);
+        this.flushTimers.set(fkey, t);
+    }
+    flushCollectionNow(key) {
+        const c = this.collections.get(key);
+        if (c && c[DIRTY]) {
+            writeJson(c.path, c.data);
+            c[DIRTY] = false;
+        }
+    }
+    flushMetaNow(key) {
+        const m = this.metadata.get(key);
+        if (m && m[DIRTY]) {
+            writeJson(m.path, m.data);
+            m[DIRTY] = false;
+        }
+    }
+    /**
+     * Flush every dirty file immediately and clear timers.
+     */
+    async flushAll() {
+        for (const t of this.flushTimers.values())
+            clearTimeout(t);
+        this.flushTimers.clear();
+        for (const key of this.collections.keys())
+            this.flushCollectionNow(key);
+        for (const key of this.metadata.keys())
+            this.flushMetaNow(key);
+    }
+    get mocksDir() {
+        return MOCKS_DIR;
+    }
+}
+export const store = new InMemoryStore();

package/dist/storage/fs.d.ts

@@ -0,0 +1,6 @@
+export declare function ensureDir(dir: string): void;
+export declare function isDir(p: string): boolean;
+export declare function readJson<T = any>(p: string, fallback: T): T;
+export declare function writeJson(p: string, data: unknown): void;
+export declare function listJsonFiles(dir: string): string[];
+export declare function basenameNoExt(p: string): string;

package/dist/storage/fs.js

@@ -0,0 +1,54 @@
+import fs from "node:fs";
+import path from "node:path";
+export function ensureDir(dir) {
+    fs.mkdirSync(dir, { recursive: true });
+}
+export function isDir(p) {
+    try {
+        return fs.statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+export function readJson(p, fallback) {
+    try {
+        const raw = fs.readFileSync(p, "utf8");
+        if (!raw.trim())
+            throw new Error("file is empty");
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return fallback;
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to read JSON fixture ${p}: ${message}`);
+    }
+}
+export function writeJson(p, data) {
+    ensureDir(path.dirname(p));
+    const serialized = JSON.stringify(data, null, 2);
+    if (serialized === undefined) {
+        throw new Error(`Cannot serialize undefined JSON fixture: ${p}`);
+    }
+    const temp = path.join(path.dirname(p), `.${path.basename(p)}.${process.pid}.tmp`);
+    try {
+        fs.writeFileSync(temp, serialized + "\n", "utf8");
+        fs.renameSync(temp, p);
+    }
+    finally {
+        if (fs.existsSync(temp))
+            fs.unlinkSync(temp);
+    }
+}
+export function listJsonFiles(dir) {
+    if (!isDir(dir))
+        return [];
+    return fs
+        .readdirSync(dir)
+        .filter((f) => f.endsWith(".json"))
+        .map((f) => path.join(dir, f));
+}
+export function basenameNoExt(p) {
+    return path.basename(p, ".json");
+}

package/dist/types.d.ts

@@ -0,0 +1,88 @@
+export type ContentType = "snapshots" | "analysis" | "decisions" | "events" | "memory";
+export declare const CONTENT_TYPES: ContentType[];
+/**
+ * The fourth tier of data alongside entities, events, and knowledge.
+ * `memory` collections are keyed by `memoryId`.
+ */
+export declare const MEMORY_CONTENT_TYPE: ContentType;
+/** Primary identity keys accepted across fetch/write engines. */
+export declare const IDENTITY_KEYS: readonly ["recordId", "entityId", "eventId", "knowledgeId", "memoryId"];
+export type IdentityKey = (typeof IDENTITY_KEYS)[number];
+/** Map a target type to its canonical id field. */
+export declare const TARGET_TO_ID_FIELD: Record<string, IdentityKey>;
+export declare const INCLUDE_FLAGS: readonly ["data", "analytics", "enrichment", "insights", "system", "associated", "discovery", "analysis"];
+export type IncludeFlag = (typeof INCLUDE_FLAGS)[number];
+export interface ExplorerQueryParams {
+    filter?: string | string[];
+    sort?: string | string[];
+    searchText?: string;
+    q?: string;
+    limit?: string | number;
+    offset?: string | number;
+    skip?: string | number;
+    includeTotal?: boolean;
+    include?: string | string[];
+    contentType?: string;
+    mode?: "descriptor" | "raw" | "auto";
+    listId?: string;
+    entityName?: string;
+    sourceLens?: "db-first" | "catalox-first";
+    target?: "entity" | "event" | "knowledge" | "memory" | string;
+    includeExactCounts?: boolean;
+    includeInventory?: boolean | string;
+    memoryId?: string;
+    recordId?: string;
+    entityId?: string;
+    eventId?: string;
+    knowledgeId?: string;
+}
+export interface CatalogRelation {
+    sourceProperty?: string;
+    targetObject?: string;
+    targetProperty?: string;
+    relationType?: string;
+}
+export interface ObjectTypeMeta {
+    name: string;
+    summary?: string;
+    rootPropertyCatalog?: RootPropertyCatalogEntry[];
+    catalogRelations?: CatalogRelation[];
+    [key: string]: unknown;
+}
+export interface RootPropertyCatalogEntry {
+    path: string;
+    occurrencePct?: number;
+    types?: string[];
+    sampleValues?: unknown[];
+    [key: string]: unknown;
+}
+export interface ListDescriptor {
+    id?: string;
+    listId?: string;
+    entity?: string;
+    entityName?: string;
+    fields?: string[];
+    projections?: Record<string, unknown>;
+    extensions?: string[];
+    sort?: Record<string, 1 | -1>;
+    query?: Record<string, unknown>;
+    [key: string]: unknown;
+}
+export interface WriteDescriptor {
+    writeDescriptorId?: string;
+    id?: string;
+    targetCollection?: string;
+    schema?: unknown;
+    [key: string]: unknown;
+}
+export interface NarrativeDefinition {
+    key: string;
+    label?: string;
+    description?: string;
+    [key: string]: unknown;
+}
+export interface AgentEntry {
+    id?: string;
+    name?: string;
+    [key: string]: unknown;
+}

package/dist/types.js

@@ -0,0 +1,37 @@
+export const CONTENT_TYPES = [
+    "snapshots",
+    "analysis",
+    "decisions",
+    "events",
+    "memory",
+];
+/**
+ * The fourth tier of data alongside entities, events, and knowledge.
+ * `memory` collections are keyed by `memoryId`.
+ */
+export const MEMORY_CONTENT_TYPE = "memory";
+/** Primary identity keys accepted across fetch/write engines. */
+export const IDENTITY_KEYS = [
+    "recordId",
+    "entityId",
+    "eventId",
+    "knowledgeId",
+    "memoryId",
+];
+/** Map a target type to its canonical id field. */
+export const TARGET_TO_ID_FIELD = {
+    entity: "recordId",
+    event: "eventId",
+    knowledge: "knowledgeId",
+    memory: "memoryId",
+};
+export const INCLUDE_FLAGS = [
+    "data",
+    "analytics",
+    "enrichment",
+    "insights",
+    "system",
+    "associated",
+    "discovery",
+    "analysis",
+];