@danielsimonjr/memoryjs 2.5.0 → 2.7.0

package/dist/index.js

@@ -3413,6 +3413,144 @@ var init_WorkerPoolManager = __esm({
   }
 });
 
+// src/utils/WorkerTaskManager.ts
+function getWorkerTaskManager() {
+  if (!DEFAULT_INSTANCE) DEFAULT_INSTANCE = new WorkerTaskManager();
+  return DEFAULT_INSTANCE;
+}
+async function batchProcessViaWorkers(items, workerType, methodName, mapArgs, opts = {}) {
+  if (items.length === 0) return [];
+  const wtm = getWorkerTaskManager();
+  const promises = items.map(
+    (item, i) => wtm.submit(workerType, methodName, mapArgs(item, i), opts)
+  );
+  return Promise.all(promises);
+}
+var SUBMISSION_COUNTER, WorkerTaskManager, DEFAULT_INSTANCE;
+var init_WorkerTaskManager = __esm({
+  "src/utils/WorkerTaskManager.ts"() {
+    "use strict";
+    init_esm_shims();
+    init_taskScheduler();
+    init_WorkerPoolManager();
+    init_logger();
+    SUBMISSION_COUNTER = 0;
+    WorkerTaskManager = class {
+      queue;
+      poolManager;
+      touchedPoolIds = /* @__PURE__ */ new Set();
+      cancelledTaskIds = /* @__PURE__ */ new Set();
+      handleStatus = /* @__PURE__ */ new Map();
+      constructor(opts = {}) {
+        this.queue = new TaskQueue({
+          concurrency: opts.concurrency,
+          timeout: opts.defaultTimeout,
+          // We dispatch to WorkerPoolManager ourselves — the queue's
+          // internal-worker-pool path would double-up.
+          useWorkerPool: false
+        });
+        this.poolManager = getWorkerPoolManager();
+      }
+      /**
+       * Submit a task and await its result. Throws on failure / cancellation.
+       *
+       * @param workerType — routing key. Maps to a named `WorkerPoolManager` pool.
+       * @param methodName — function name exposed by the worker module.
+       * @param args — positional arguments passed to the worker function.
+       */
+      async submit(workerType, methodName, args, opts = {}) {
+        const handle = this.submitWithHandle(workerType, methodName, args, opts);
+        return handle.result;
+      }
+      /**
+       * Submit a task and return a handle for cancellation + status polling.
+       */
+      submitWithHandle(workerType, methodName, args, opts = {}) {
+        const id = `wtm-${Date.now()}-${++SUBMISSION_COUNTER}`;
+        this.handleStatus.set(id, "pending" /* PENDING */);
+        const task = {
+          id,
+          priority: opts.priority ?? 1 /* NORMAL */,
+          timeout: opts.timeout,
+          input: { workerType, methodName, args },
+          fn: async (input) => {
+            if (this.cancelledTaskIds.has(id)) {
+              throw new Error(`Task ${id} cancelled before dispatch`);
+            }
+            this.handleStatus.set(id, "running" /* RUNNING */);
+            const pool = this.poolManager.getPool(input.workerType, opts.poolConfig);
+            this.touchedPoolIds.add(input.workerType);
+            return await pool.exec(input.methodName, input.args);
+          }
+        };
+        const result = this.queue.enqueue(task).then(async (res) => {
+          this.handleStatus.set(id, res.status);
+          if (res.status === "completed" /* COMPLETED */) {
+            return res.result;
+          }
+          if (res.status === "cancelled" /* CANCELLED */ || this.cancelledTaskIds.has(id)) {
+            throw new Error(`Task ${id} cancelled`);
+          }
+          const err2 = res.error ?? new Error(`Task ${id} failed: ${res.status}`);
+          throw err2;
+        });
+        return {
+          id,
+          result,
+          cancel: () => {
+            this.cancelledTaskIds.add(id);
+            const evicted = this.queue.cancel(id);
+            if (evicted) {
+              this.handleStatus.set(id, "cancelled" /* CANCELLED */);
+            }
+            return evicted;
+          },
+          status: () => this.handleStatus.get(id) ?? "pending" /* PENDING */
+        };
+      }
+      /**
+       * Aggregated stats across the queue + touched pools.
+       */
+      getStats() {
+        const pools = [];
+        for (const poolId of this.touchedPoolIds) {
+          const stats = this.poolManager.getPoolStats(poolId);
+          if (stats) {
+            pools.push({
+              poolId,
+              workers: stats.totalWorkers,
+              activeTasks: stats.activeTasks,
+              pendingTasks: stats.pendingTasks,
+              totalTasksExecuted: stats.totalTasksExecuted
+            });
+          }
+        }
+        return { queue: this.queue.getStats(), pools };
+      }
+      /**
+       * Drain the queue (wait for all pending + running tasks) and return their
+       * results in completion order. Pool workers are NOT terminated — call
+       * `shutdown()` for that.
+       */
+      async drain() {
+        return this.queue.drain();
+      }
+      /**
+       * Shut down the queue + the underlying WorkerPoolManager pools touched by
+       * this instance. Idempotent. Returns after every owned worker has exited.
+       */
+      async shutdown() {
+        try {
+          await this.queue.shutdown();
+        } catch (e) {
+          logger.warn("[WorkerTaskManager.shutdown] queue.shutdown failed:", e);
+        }
+      }
+    };
+    DEFAULT_INSTANCE = null;
+  }
+});
+
 // src/utils/BatchProcessor.ts
 async function processBatch(items, processor, options = {}) {
   const batchProcessor = new BatchProcessor(options);
@@ -11267,6 +11405,7 @@ var init_utils = __esm({
     init_taskScheduler();
     init_operationUtils();
     init_WorkerPoolManager();
+    init_WorkerTaskManager();
     init_BatchProcessor();
     init_MemoryMonitor();
     init_relationHelpers();
@@ -20623,6 +20762,431 @@ var ObservationStore = class _ObservationStore {
 
 // src/core/StorageFactory.ts
 init_esm_shims();
+
+// src/core/PostgreSQLStorage.ts
+init_esm_shims();
+init_logger();
+var ENTITY_COLUMNS = [
+  "name",
+  "entity_type",
+  "observations",
+  "parent_id",
+  "tags",
+  "importance",
+  "created_at",
+  "last_modified",
+  "ttl",
+  "confidence",
+  "project_id",
+  "version",
+  "parent_entity_name",
+  "root_entity_name",
+  "is_latest",
+  "superseded_by",
+  "content_hash",
+  "valid_from",
+  "valid_until",
+  "observation_meta",
+  "lifecycle_status",
+  "extra"
+];
+var FIRST_CLASS_ENTITY_KEYS = /* @__PURE__ */ new Set([
+  "name",
+  "entityType",
+  "observations",
+  "parentId",
+  "tags",
+  "importance",
+  "createdAt",
+  "lastModified",
+  "ttl",
+  "confidence",
+  "projectId",
+  "version",
+  "parentEntityName",
+  "rootEntityName",
+  "isLatest",
+  "supersededBy",
+  "contentHash",
+  "validFrom",
+  "validUntil",
+  "observationMeta",
+  "lifecycleStatus"
+]);
+var SCHEMA_DDL = `
+CREATE TABLE IF NOT EXISTS entities (
+  name TEXT PRIMARY KEY,
+  entity_type TEXT NOT NULL,
+  observations TEXT[] NOT NULL DEFAULT '{}',
+  parent_id TEXT,
+  tags TEXT[],
+  importance NUMERIC(4, 2),
+  created_at TIMESTAMPTZ DEFAULT NOW(),
+  last_modified TIMESTAMPTZ DEFAULT NOW(),
+  ttl INTEGER,
+  confidence NUMERIC(4, 3),
+  project_id TEXT,
+  version INTEGER,
+  parent_entity_name TEXT,
+  root_entity_name TEXT,
+  is_latest BOOLEAN,
+  superseded_by TEXT,
+  content_hash TEXT,
+  valid_from TIMESTAMPTZ,
+  valid_until TIMESTAMPTZ,
+  observation_meta JSONB,
+  lifecycle_status TEXT,
+  extra JSONB
+);
+
+CREATE INDEX IF NOT EXISTS idx_entities_entity_type ON entities(entity_type);
+CREATE INDEX IF NOT EXISTS idx_entities_project_id ON entities(project_id);
+CREATE INDEX IF NOT EXISTS idx_entities_content_hash ON entities(content_hash);
+CREATE INDEX IF NOT EXISTS idx_entities_tags_gin ON entities USING GIN(tags);
+
+CREATE TABLE IF NOT EXISTS relations (
+  from_name TEXT NOT NULL,
+  to_name TEXT NOT NULL,
+  relation_type TEXT NOT NULL,
+  metadata JSONB,
+  created_at TIMESTAMPTZ DEFAULT NOW(),
+  valid_from TIMESTAMPTZ,
+  valid_until TIMESTAMPTZ,
+  PRIMARY KEY (from_name, to_name, relation_type)
+);
+
+CREATE INDEX IF NOT EXISTS idx_relations_to ON relations(to_name);
+CREATE INDEX IF NOT EXISTS idx_relations_type ON relations(relation_type);
+`;
+var PostgreSQLStorage = class {
+  connectionString;
+  pool = null;
+  cache = null;
+  nameIndex = /* @__PURE__ */ new Map();
+  outgoingRelations = /* @__PURE__ */ new Map();
+  incomingRelations = /* @__PURE__ */ new Map();
+  pendingAppends = 0;
+  schemaInitPromise = null;
+  constructor(connectionString) {
+    this.connectionString = connectionString;
+  }
+  // ==================== Connection management ====================
+  /**
+   * Get or lazily-construct the `pg.Pool`. Throws a friendly error if
+   * the `pg` package isn't installed.
+   */
+  async getPool() {
+    if (this.pool) return this.pool;
+    let pgModule;
+    try {
+      pgModule = await import("pg");
+    } catch {
+      throw new Error(
+        "The 'pg' package is required for the PostgreSQL backend but is not installed. Run: npm install pg @types/pg"
+      );
+    }
+    this.pool = new pgModule.Pool({ connectionString: this.connectionString });
+    return this.pool;
+  }
+  /**
+   * Run schema DDL idempotently. Called from `ensureLoaded`.
+   */
+  async initSchema() {
+    if (this.schemaInitPromise) return this.schemaInitPromise;
+    this.schemaInitPromise = (async () => {
+      const pool = await this.getPool();
+      await pool.query(SCHEMA_DDL);
+    })();
+    return this.schemaInitPromise;
+  }
+  // ==================== Row ⇄ Entity / Relation mapping ====================
+  entityToRow(entity) {
+    const extra = {};
+    for (const key of Object.keys(entity)) {
+      if (!FIRST_CLASS_ENTITY_KEYS.has(key)) {
+        extra[key] = entity[key];
+      }
+    }
+    return {
+      name: entity.name,
+      entity_type: entity.entityType,
+      observations: entity.observations,
+      parent_id: entity.parentId ?? null,
+      tags: entity.tags ?? null,
+      importance: entity.importance ?? null,
+      created_at: entity.createdAt ?? null,
+      last_modified: entity.lastModified ?? null,
+      ttl: entity.ttl ?? null,
+      confidence: entity.confidence ?? null,
+      project_id: entity.projectId ?? null,
+      version: entity.version ?? null,
+      parent_entity_name: entity.parentEntityName ?? null,
+      root_entity_name: entity.rootEntityName ?? null,
+      is_latest: entity.isLatest ?? null,
+      superseded_by: entity.supersededBy ?? null,
+      content_hash: entity.contentHash ?? null,
+      valid_from: entity.validFrom ?? null,
+      valid_until: entity.validUntil ?? null,
+      observation_meta: entity.observationMeta ?? null,
+      lifecycle_status: entity.lifecycleStatus ?? null,
+      extra: Object.keys(extra).length > 0 ? extra : null
+    };
+  }
+  rowToEntity(row) {
+    const entity = {
+      name: String(row.name),
+      entityType: String(row.entity_type),
+      observations: Array.isArray(row.observations) ? row.observations : []
+    };
+    if (row.parent_id != null) entity.parentId = String(row.parent_id);
+    if (Array.isArray(row.tags)) entity.tags = row.tags;
+    if (row.importance != null) entity.importance = Number(row.importance);
+    if (row.created_at != null) entity.createdAt = new Date(String(row.created_at)).toISOString();
+    if (row.last_modified != null) entity.lastModified = new Date(String(row.last_modified)).toISOString();
+    if (row.ttl != null) entity.ttl = Number(row.ttl);
+    if (row.confidence != null) entity.confidence = Number(row.confidence);
+    if (row.project_id != null) entity.projectId = String(row.project_id);
+    if (row.version != null) entity.version = Number(row.version);
+    if (row.parent_entity_name != null) entity.parentEntityName = String(row.parent_entity_name);
+    if (row.root_entity_name != null) entity.rootEntityName = String(row.root_entity_name);
+    if (row.is_latest != null) entity.isLatest = Boolean(row.is_latest);
+    if (row.superseded_by != null) entity.supersededBy = String(row.superseded_by);
+    if (row.content_hash != null) entity.contentHash = String(row.content_hash);
+    if (row.valid_from != null) entity.validFrom = new Date(String(row.valid_from)).toISOString();
+    if (row.valid_until != null) entity.validUntil = new Date(String(row.valid_until)).toISOString();
+    if (row.observation_meta != null) {
+      entity.observationMeta = row.observation_meta;
+    }
+    if (row.lifecycle_status != null) {
+      entity.lifecycleStatus = String(row.lifecycle_status);
+    }
+    if (row.extra != null && typeof row.extra === "object") {
+      Object.assign(entity, row.extra);
+    }
+    return entity;
+  }
+  rowToRelation(row) {
+    const relation = {
+      from: String(row.from_name),
+      to: String(row.to_name),
+      relationType: String(row.relation_type)
+    };
+    return relation;
+  }
+  // ==================== Cache management ====================
+  rebuildIndexes(graph) {
+    this.nameIndex.clear();
+    this.outgoingRelations.clear();
+    this.incomingRelations.clear();
+    for (const e of graph.entities) this.nameIndex.set(e.name, e);
+    for (const r of graph.relations) {
+      const outArr = this.outgoingRelations.get(r.from) ?? [];
+      outArr.push(r);
+      this.outgoingRelations.set(r.from, outArr);
+      const inArr = this.incomingRelations.get(r.to) ?? [];
+      inArr.push(r);
+      this.incomingRelations.set(r.to, inArr);
+    }
+  }
+  // ==================== IGraphStorage — read ====================
+  async loadGraph() {
+    await this.ensureLoaded();
+    return this.cache;
+  }
+  async getGraphForMutation() {
+    await this.ensureLoaded();
+    const cache = this.cache;
+    return {
+      entities: cache.entities.map((e) => ({
+        ...e,
+        observations: [...e.observations],
+        tags: e.tags ? [...e.tags] : void 0
+      })),
+      relations: cache.relations.map((r) => ({ ...r }))
+    };
+  }
+  async ensureLoaded() {
+    if (this.cache) return;
+    await this.initSchema();
+    const pool = await this.getPool();
+    const [eRes, rRes] = await Promise.all([
+      pool.query("SELECT * FROM entities"),
+      pool.query("SELECT * FROM relations")
+    ]);
+    const graph = {
+      entities: eRes.rows.map((r) => this.rowToEntity(r)),
+      relations: rRes.rows.map((r) => this.rowToRelation(r))
+    };
+    this.cache = graph;
+    this.rebuildIndexes(graph);
+  }
+  get cachedGraph() {
+    return this.cache;
+  }
+  // ==================== IGraphStorage — write ====================
+  async saveGraph(graph) {
+    await this.initSchema();
+    const pool = await this.getPool();
+    await pool.query("TRUNCATE entities, relations");
+    for (const entity of graph.entities) {
+      await this.insertEntityRow(pool, entity);
+    }
+    for (const relation of graph.relations) {
+      await this.insertRelationRow(pool, relation);
+    }
+    this.cache = {
+      entities: graph.entities.map((e) => ({ ...e })),
+      relations: graph.relations.map((r) => ({ ...r }))
+    };
+    this.rebuildIndexes(this.cache);
+    this.pendingAppends = 0;
+  }
+  async insertEntityRow(pool, entity) {
+    const row = this.entityToRow(entity);
+    const cols = ENTITY_COLUMNS.join(", ");
+    const placeholders = ENTITY_COLUMNS.map((_, i) => `$${i + 1}`).join(", ");
+    const values = ENTITY_COLUMNS.map((c) => row[c]);
+    await pool.query(
+      `INSERT INTO entities (${cols}) VALUES (${placeholders})
+       ON CONFLICT (name) DO UPDATE SET ${ENTITY_COLUMNS.filter((c) => c !== "name").map((c) => `${c} = EXCLUDED.${c}`).join(", ")}`,
+      values
+    );
+  }
+  async insertRelationRow(pool, relation) {
+    await pool.query(
+      `INSERT INTO relations (from_name, to_name, relation_type)
+       VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`,
+      [relation.from, relation.to, relation.relationType]
+    );
+  }
+  async appendEntity(entity) {
+    await this.ensureLoaded();
+    const pool = await this.getPool();
+    await this.insertEntityRow(pool, entity);
+    if (!this.cache) return;
+    const existingIdx = this.cache.entities.findIndex((e) => e.name === entity.name);
+    if (existingIdx >= 0) this.cache.entities[existingIdx] = entity;
+    else this.cache.entities.push(entity);
+    this.nameIndex.set(entity.name, entity);
+    this.pendingAppends += 1;
+  }
+  async appendRelation(relation) {
+    await this.ensureLoaded();
+    const pool = await this.getPool();
+    await this.insertRelationRow(pool, relation);
+    if (!this.cache) return;
+    const duplicate = this.cache.relations.some(
+      (r) => r.from === relation.from && r.to === relation.to && r.relationType === relation.relationType
+    );
+    if (!duplicate) {
+      this.cache.relations.push(relation);
+      const outArr = this.outgoingRelations.get(relation.from) ?? [];
+      outArr.push(relation);
+      this.outgoingRelations.set(relation.from, outArr);
+      const inArr = this.incomingRelations.get(relation.to) ?? [];
+      inArr.push(relation);
+      this.incomingRelations.set(relation.to, inArr);
+    }
+    this.pendingAppends += 1;
+  }
+  async updateEntity(entityName, updates) {
+    await this.ensureLoaded();
+    const existing = this.nameIndex.get(entityName);
+    if (!existing) return false;
+    const merged = {
+      ...existing,
+      ...updates,
+      name: entityName,
+      lastModified: updates.lastModified ?? (/* @__PURE__ */ new Date()).toISOString()
+    };
+    const pool = await this.getPool();
+    await this.insertEntityRow(pool, merged);
+    if (this.cache) {
+      const idx = this.cache.entities.findIndex((e) => e.name === entityName);
+      if (idx >= 0) this.cache.entities[idx] = merged;
+    }
+    this.nameIndex.set(entityName, merged);
+    this.pendingAppends += 1;
+    return true;
+  }
+  async compact() {
+    if (this.cache) await this.saveGraph(this.cache);
+  }
+  clearCache() {
+    this.cache = null;
+    this.nameIndex.clear();
+    this.outgoingRelations.clear();
+    this.incomingRelations.clear();
+    this.pendingAppends = 0;
+  }
+  // ==================== IGraphStorage — sync getters ====================
+  getEntityByName(name) {
+    return this.nameIndex.get(name);
+  }
+  hasEntity(name) {
+    return this.nameIndex.has(name);
+  }
+  getEntitiesByType(entityType) {
+    const result = [];
+    for (const entity of this.nameIndex.values()) {
+      if (entity.entityType === entityType) result.push(entity);
+    }
+    return result;
+  }
+  getEntityTypes() {
+    const types = /* @__PURE__ */ new Set();
+    for (const entity of this.nameIndex.values()) types.add(entity.entityType);
+    return Array.from(types);
+  }
+  getLowercased(entityName) {
+    const e = this.nameIndex.get(entityName);
+    if (!e) return void 0;
+    return {
+      name: e.name.toLowerCase(),
+      entityType: e.entityType.toLowerCase(),
+      observations: e.observations.map((o) => o.toLowerCase()),
+      tags: e.tags?.map((t) => t.toLowerCase()) ?? []
+    };
+  }
+  getRelationsFrom(entityName) {
+    return this.outgoingRelations.get(entityName) ?? [];
+  }
+  getRelationsTo(entityName) {
+    return this.incomingRelations.get(entityName) ?? [];
+  }
+  getRelationsFor(entityName) {
+    return [
+      ...this.outgoingRelations.get(entityName) ?? [],
+      ...this.incomingRelations.get(entityName) ?? []
+    ];
+  }
+  hasRelations(entityName) {
+    return (this.outgoingRelations.get(entityName)?.length ?? 0) + (this.incomingRelations.get(entityName)?.length ?? 0) > 0;
+  }
+  // ==================== Utility ====================
+  getFilePath() {
+    return this.connectionString;
+  }
+  getPendingAppends() {
+    return this.pendingAppends;
+  }
+  /**
+   * Close the underlying `pg.Pool`. Call from process-shutdown handlers to
+   * avoid keeping the event loop alive on idle connections.
+   */
+  async close() {
+    if (this.pool) {
+      try {
+        await this.pool.end();
+      } catch (e) {
+        logger.warn("PostgreSQLStorage.close: pool.end failed:", e);
+      }
+      this.pool = null;
+    }
+  }
+};
+
+// src/core/StorageFactory.ts
 var DEFAULT_STORAGE_TYPE = "jsonl";
 function createStorage(config) {
   const storageType = process.env.MEMORY_STORAGE_TYPE || config.type || DEFAULT_STORAGE_TYPE;
@@ -20631,9 +21195,12 @@ function createStorage(config) {
       return new GraphStorage(config.path);
     case "sqlite":
       return new SQLiteStorage(config.path);
+    case "postgres":
+    case "postgresql":
+      return new PostgreSQLStorage(config.path);
     default:
       throw new Error(
-        `Unknown storage type: ${storageType}. Supported types: jsonl, sqlite`
+        `Unknown storage type: ${storageType}. Supported types: jsonl, sqlite, postgres`
       );
   }
 }
@@ -51333,6 +51900,7 @@ export {
   VisibilityResolver,
   WorkThreadManager,
   WorkerPoolManager,
+  WorkerTaskManager,
   WorkingMemoryManager,
   WorldModelManager,
   WorldStateSnapshot,
@@ -51342,6 +51910,7 @@ export {
   applyPagination,
   asWarning,
   batchProcess,
+  batchProcessViaWorkers,
   buildTFVector,
   calculateIDF,
   calculateIDFFromTokenSets,
@@ -51412,6 +51981,7 @@ export {
   getQuickHint,
   getRoleProfile,
   getWorkerPoolManager,
+  getWorkerTaskManager,
   globalMemoryMonitor,
   groupEntitiesByType,
   hasAllTags,