@danielsimonjr/memoryjs 2.6.0 → 2.8.0

package/dist/index.js

@@ -3413,6 +3413,144 @@ var init_WorkerPoolManager = __esm({
   }
 });
 
+// src/utils/WorkerTaskManager.ts
+function getWorkerTaskManager() {
+  if (!DEFAULT_INSTANCE) DEFAULT_INSTANCE = new WorkerTaskManager();
+  return DEFAULT_INSTANCE;
+}
+async function batchProcessViaWorkers(items, workerType, methodName, mapArgs, opts = {}) {
+  if (items.length === 0) return [];
+  const wtm = getWorkerTaskManager();
+  const promises = items.map(
+    (item, i) => wtm.submit(workerType, methodName, mapArgs(item, i), opts)
+  );
+  return Promise.all(promises);
+}
+var SUBMISSION_COUNTER, WorkerTaskManager, DEFAULT_INSTANCE;
+var init_WorkerTaskManager = __esm({
+  "src/utils/WorkerTaskManager.ts"() {
+    "use strict";
+    init_esm_shims();
+    init_taskScheduler();
+    init_WorkerPoolManager();
+    init_logger();
+    SUBMISSION_COUNTER = 0;
+    WorkerTaskManager = class {
+      queue;
+      poolManager;
+      touchedPoolIds = /* @__PURE__ */ new Set();
+      cancelledTaskIds = /* @__PURE__ */ new Set();
+      handleStatus = /* @__PURE__ */ new Map();
+      constructor(opts = {}) {
+        this.queue = new TaskQueue({
+          concurrency: opts.concurrency,
+          timeout: opts.defaultTimeout,
+          // We dispatch to WorkerPoolManager ourselves — the queue's
+          // internal-worker-pool path would double-up.
+          useWorkerPool: false
+        });
+        this.poolManager = getWorkerPoolManager();
+      }
+      /**
+       * Submit a task and await its result. Throws on failure / cancellation.
+       *
+       * @param workerType — routing key. Maps to a named `WorkerPoolManager` pool.
+       * @param methodName — function name exposed by the worker module.
+       * @param args — positional arguments passed to the worker function.
+       */
+      async submit(workerType, methodName, args, opts = {}) {
+        const handle = this.submitWithHandle(workerType, methodName, args, opts);
+        return handle.result;
+      }
+      /**
+       * Submit a task and return a handle for cancellation + status polling.
+       */
+      submitWithHandle(workerType, methodName, args, opts = {}) {
+        const id = `wtm-${Date.now()}-${++SUBMISSION_COUNTER}`;
+        this.handleStatus.set(id, "pending" /* PENDING */);
+        const task = {
+          id,
+          priority: opts.priority ?? 1 /* NORMAL */,
+          timeout: opts.timeout,
+          input: { workerType, methodName, args },
+          fn: async (input) => {
+            if (this.cancelledTaskIds.has(id)) {
+              throw new Error(`Task ${id} cancelled before dispatch`);
+            }
+            this.handleStatus.set(id, "running" /* RUNNING */);
+            const pool = this.poolManager.getPool(input.workerType, opts.poolConfig);
+            this.touchedPoolIds.add(input.workerType);
+            return await pool.exec(input.methodName, input.args);
+          }
+        };
+        const result = this.queue.enqueue(task).then(async (res) => {
+          this.handleStatus.set(id, res.status);
+          if (res.status === "completed" /* COMPLETED */) {
+            return res.result;
+          }
+          if (res.status === "cancelled" /* CANCELLED */ || this.cancelledTaskIds.has(id)) {
+            throw new Error(`Task ${id} cancelled`);
+          }
+          const err2 = res.error ?? new Error(`Task ${id} failed: ${res.status}`);
+          throw err2;
+        });
+        return {
+          id,
+          result,
+          cancel: () => {
+            this.cancelledTaskIds.add(id);
+            const evicted = this.queue.cancel(id);
+            if (evicted) {
+              this.handleStatus.set(id, "cancelled" /* CANCELLED */);
+            }
+            return evicted;
+          },
+          status: () => this.handleStatus.get(id) ?? "pending" /* PENDING */
+        };
+      }
+      /**
+       * Aggregated stats across the queue + touched pools.
+       */
+      getStats() {
+        const pools = [];
+        for (const poolId of this.touchedPoolIds) {
+          const stats = this.poolManager.getPoolStats(poolId);
+          if (stats) {
+            pools.push({
+              poolId,
+              workers: stats.totalWorkers,
+              activeTasks: stats.activeTasks,
+              pendingTasks: stats.pendingTasks,
+              totalTasksExecuted: stats.totalTasksExecuted
+            });
+          }
+        }
+        return { queue: this.queue.getStats(), pools };
+      }
+      /**
+       * Drain the queue (wait for all pending + running tasks) and return their
+       * results in completion order. Pool workers are NOT terminated — call
+       * `shutdown()` for that.
+       */
+      async drain() {
+        return this.queue.drain();
+      }
+      /**
+       * Shut down the queue + the underlying WorkerPoolManager pools touched by
+       * this instance. Idempotent. Returns after every owned worker has exited.
+       */
+      async shutdown() {
+        try {
+          await this.queue.shutdown();
+        } catch (e) {
+          logger.warn("[WorkerTaskManager.shutdown] queue.shutdown failed:", e);
+        }
+      }
+    };
+    DEFAULT_INSTANCE = null;
+  }
+});
+
 // src/utils/BatchProcessor.ts
 async function processBatch(items, processor, options = {}) {
   const batchProcessor = new BatchProcessor(options);
@@ -11267,6 +11405,7 @@ var init_utils = __esm({
     init_taskScheduler();
     init_operationUtils();
     init_WorkerPoolManager();
+    init_WorkerTaskManager();
     init_BatchProcessor();
     init_MemoryMonitor();
     init_relationHelpers();
@@ -20705,6 +20844,27 @@ CREATE INDEX IF NOT EXISTS idx_entities_project_id ON entities(project_id);
 CREATE INDEX IF NOT EXISTS idx_entities_content_hash ON entities(content_hash);
 CREATE INDEX IF NOT EXISTS idx_entities_tags_gin ON entities USING GIN(tags);
 
+-- v2.8.0 \u2014 generated tsvector column for full-text search.
+-- name is weighted A (highest), observations B, tags C. PostgreSQL 12+
+-- supports GENERATED ALWAYS AS ... STORED; the DO block makes the column
+-- add idempotent on older + newer servers without requiring CREATE OR
+-- REPLACE semantics on ALTER.
+DO $$ BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_name = 'entities' AND column_name = 'fts_vector'
+  ) THEN
+    ALTER TABLE entities ADD COLUMN fts_vector tsvector
+      GENERATED ALWAYS AS (
+        setweight(to_tsvector('english', name), 'A') ||
+        setweight(to_tsvector('english', coalesce(array_to_string(observations, ' '), '')), 'B') ||
+        setweight(to_tsvector('english', coalesce(array_to_string(tags, ' '), '')), 'C')
+      ) STORED;
+  END IF;
+END $$;
+
+CREATE INDEX IF NOT EXISTS idx_entities_fts_gin ON entities USING GIN(fts_vector);
+
 CREATE TABLE IF NOT EXISTS relations (
   from_name TEXT NOT NULL,
   to_name TEXT NOT NULL,
@@ -21024,6 +21184,38 @@ var PostgreSQLStorage = class {
   hasRelations(entityName) {
     return (this.outgoingRelations.get(entityName)?.length ?? 0) + (this.incomingRelations.get(entityName)?.length ?? 0) > 0;
   }
+  // ==================== Full-text search ====================
+  /**
+   * tsvector-backed full-text search. Uses `plainto_tsquery` so the input is
+   * free-form (no boolean operator syntax required); ranks via `ts_rank` on
+   * the weighted `fts_vector` column (name × A, observations × B, tags × C).
+   *
+   * Results are ordered by descending score and capped at `options.limit`
+   * (default 50). Empty / whitespace-only queries return `[]` without
+   * issuing a SQL call.
+   *
+   * @example
+   * ```typescript
+   * const matches = await storage.fullTextSearch('authentication flow', { limit: 10 });
+   * // → [{ name: 'AuthService', score: 0.62 }, ...]
+   * ```
+   */
+  async fullTextSearch(query, options = {}) {
+    const trimmed = query.trim();
+    if (trimmed.length === 0) return [];
+    await this.initSchema();
+    const pool = await this.getPool();
+    const limit = options.limit ?? 50;
+    const res = await pool.query(
+      `SELECT name, ts_rank(fts_vector, plainto_tsquery('english', $1))::float AS score
+       FROM entities
+       WHERE fts_vector @@ plainto_tsquery('english', $1)
+       ORDER BY score DESC
+       LIMIT $2`,
+      [trimmed, limit]
+    );
+    return res.rows.map((r) => ({ name: r.name, score: Number(r.score) }));
+  }
   // ==================== Utility ====================
   getFilePath() {
     return this.connectionString;
@@ -51761,6 +51953,7 @@ export {
   VisibilityResolver,
   WorkThreadManager,
   WorkerPoolManager,
+  WorkerTaskManager,
   WorkingMemoryManager,
   WorldModelManager,
   WorldStateSnapshot,
@@ -51770,6 +51963,7 @@ export {
   applyPagination,
   asWarning,
   batchProcess,
+  batchProcessViaWorkers,
   buildTFVector,
   calculateIDF,
   calculateIDFFromTokenSets,
@@ -51840,6 +52034,7 @@ export {
   getQuickHint,
   getRoleProfile,
   getWorkerPoolManager,
+  getWorkerTaskManager,
   globalMemoryMonitor,
   groupEntitiesByType,
   hasAllTags,