@stackmemoryai/stackmemory 1.10.5 → 1.14.0

package/dist/src/core/storage/cloud-sync.js

@@ -0,0 +1,574 @@
+import { fileURLToPath as __fileURLToPath } from 'url';
+import { dirname as __pathDirname } from 'path';
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+import { createHash } from "crypto";
+import { AsyncMutex } from "../utils/async-mutex.js";
+import { logger } from "../monitoring/logger.js";
+const SYNCABLE_TABLES = [
+  "frames",
+  "events",
+  "anchors",
+  "trace_events",
+  "entity_states"
+];
+const VERSION_COLUMN = {
+  frames: "created_at",
+  events: "ts",
+  anchors: "created_at",
+  trace_events: "timestamp",
+  entity_states: "valid_from"
+};
+const PK_COLUMN = {
+  frames: "frame_id",
+  events: "event_id",
+  anchors: "anchor_id",
+  trace_events: "id",
+  entity_states: "id"
+};
+const VALID_COLUMNS = {
+  frames: /* @__PURE__ */ new Set([
+    "frame_id",
+    "run_id",
+    "project_id",
+    "parent_frame_id",
+    "depth",
+    "type",
+    "name",
+    "state",
+    "inputs",
+    "outputs",
+    "digest_text",
+    "digest_json",
+    "created_at",
+    "closed_at",
+    "retention_policy",
+    "importance_score",
+    "prov_source",
+    "prov_derivation",
+    "prov_confidence",
+    "prov_superseded_by",
+    "prov_program_version",
+    "access_count",
+    "last_accessed"
+  ]),
+  events: /* @__PURE__ */ new Set([
+    "event_id",
+    "frame_id",
+    "run_id",
+    "seq",
+    "event_type",
+    "payload",
+    "ts"
+  ]),
+  anchors: /* @__PURE__ */ new Set([
+    "anchor_id",
+    "frame_id",
+    "project_id",
+    "type",
+    "text",
+    "priority",
+    "created_at",
+    "metadata",
+    "prov_source",
+    "prov_confidence",
+    "prov_superseded_by"
+  ]),
+  trace_events: /* @__PURE__ */ new Set([
+    "id",
+    "timestamp",
+    "session_id",
+    "trace_id",
+    "parent_trace_id",
+    "tenant_id",
+    "actor_host",
+    "actor_agent",
+    "actor_user",
+    "operation",
+    "inputs",
+    "outputs",
+    "tokens_in",
+    "tokens_out",
+    "cost_usd",
+    "duration_ms",
+    "score",
+    "feedback",
+    "provenance",
+    "error",
+    "tags"
+  ]),
+  entity_states: /* @__PURE__ */ new Set([
+    "id",
+    "project_id",
+    "entity_name",
+    "relation",
+    "value",
+    "context",
+    "source_frame_id",
+    "valid_from",
+    "superseded_at"
+  ])
+};
+class CloudSyncEngine {
+  db;
+  config;
+  mutex;
+  fetchFn;
+  constructor(db, config, fetchFn) {
+    this.db = db;
+    this.config = config;
+    this.mutex = new AsyncMutex(3e5);
+    this.fetchFn = fetchFn ?? globalThis.fetch;
+  }
+  /**
+   * Push local changes to cloud
+   */
+  async push(force = false) {
+    return this.mutex.withLock(async () => {
+      try {
+        this.ensureSyncTables();
+        if (force) {
+          this.db.prepare(`DELETE FROM cloud_sync_state`).run();
+          this.db.prepare(`DELETE FROM cloud_sync_cursors WHERE direction = 'push'`).run();
+        }
+        const pending = this.collectPendingEntities();
+        if (pending.length === 0) {
+          return { success: true, pushed: 0, rejected: 0, conflicts: 0 };
+        }
+        let totalPushed = 0;
+        let totalRejected = 0;
+        let totalConflicts = 0;
+        for (let i = 0; i < pending.length; i += this.config.batchSize) {
+          const batch = pending.slice(i, i + this.config.batchSize);
+          const cursor = this.getCursor("push");
+          const request = {
+            protocolVersion: 1,
+            clientId: this.config.clientId,
+            projectId: this.config.projectId,
+            syncCursor: cursor,
+            entities: batch,
+            checksum: this.computeChecksum(batch)
+          };
+          const response = await this.sendPush(request);
+          if (!response) {
+            return {
+              success: false,
+              pushed: totalPushed,
+              rejected: totalRejected,
+              conflicts: totalConflicts,
+              error: "Network error"
+            };
+          }
+          const rejectedIds = new Set(response.rejected.map((r) => r.id));
+          const conflictIds = new Set(
+            (response.conflicts ?? []).map((c) => c.id)
+          );
+          for (const entity of batch) {
+            if (conflictIds.has(entity.id)) {
+              this.updateSyncState(entity.table, entity.id, "conflict");
+            } else if (!rejectedIds.has(entity.id)) {
+              this.markPushed(entity.table, entity.id, entity.version);
+            } else {
+              this.recordPushError(
+                entity.table,
+                entity.id,
+                response.rejected.find((r) => r.id === entity.id)?.reason ?? "unknown"
+              );
+            }
+          }
+          this.setCursor("push", response.serverCursor);
+          totalPushed += response.accepted;
+          totalRejected += response.rejected.length;
+          totalConflicts += (response.conflicts ?? []).length;
+        }
+        logger.info("Cloud sync push completed", {
+          pushed: totalPushed,
+          rejected: totalRejected,
+          conflicts: totalConflicts
+        });
+        return {
+          success: true,
+          pushed: totalPushed,
+          rejected: totalRejected,
+          conflicts: totalConflicts
+        };
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        logger.error("Cloud sync push failed", { error: msg });
+        return {
+          success: false,
+          pushed: 0,
+          rejected: 0,
+          conflicts: 0,
+          error: msg
+        };
+      }
+    }, "cloud-sync-push");
+  }
+  /**
+   * Pull remote changes to local
+   */
+  async pull(tables) {
+    return this.mutex.withLock(async () => {
+      try {
+        this.ensureSyncTables();
+        let totalPulled = 0;
+        let totalApplied = 0;
+        let totalConflicts = 0;
+        let hasMore = true;
+        let cursor = this.getCursor("pull");
+        while (hasMore) {
+          const request = {
+            protocolVersion: 1,
+            clientId: this.config.clientId,
+            projectId: this.config.projectId,
+            since: cursor,
+            tables,
+            limit: this.config.batchSize
+          };
+          const response = await this.sendPull(request);
+          if (!response) {
+            return {
+              success: false,
+              pulled: totalPulled,
+              applied: totalApplied,
+              conflicts: totalConflicts,
+              error: "Network error"
+            };
+          }
+          const applied = this.applyPulledEntities(response.entities);
+          totalPulled += response.entities.length;
+          totalApplied += applied.applied;
+          totalConflicts += applied.conflicts;
+          cursor = response.serverCursor;
+          hasMore = response.hasMore;
+        }
+        this.setCursor("pull", cursor);
+        logger.info("Cloud sync pull completed", {
+          pulled: totalPulled,
+          applied: totalApplied,
+          conflicts: totalConflicts
+        });
+        return {
+          success: true,
+          pulled: totalPulled,
+          applied: totalApplied,
+          conflicts: totalConflicts
+        };
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        logger.error("Cloud sync pull failed", { error: msg });
+        return {
+          success: false,
+          pulled: 0,
+          applied: 0,
+          conflicts: 0,
+          error: msg
+        };
+      }
+    }, "cloud-sync-pull");
+  }
+  /**
+   * Get current sync status
+   */
+  status() {
+    this.ensureSyncTables();
+    const pendingPush = this.db.prepare(
+      `SELECT COUNT(*) as count FROM cloud_sync_state WHERE sync_status = 'pending'`
+    ).get();
+    const conflicts = this.db.prepare(
+      `SELECT COUNT(*) as count FROM cloud_sync_state WHERE sync_status = 'conflict'`
+    ).get();
+    const pushCursor = this.db.prepare(
+      `SELECT cursor_value FROM cloud_sync_cursors WHERE direction = 'push'`
+    ).get();
+    const pullCursor = this.db.prepare(
+      `SELECT cursor_value FROM cloud_sync_cursors WHERE direction = 'pull'`
+    ).get();
+    let untrackedCount = 0;
+    for (const table of SYNCABLE_TABLES) {
+      if (!this.tableExists(table)) continue;
+      const pk = PK_COLUMN[table];
+      const row = this.db.prepare(
+        `SELECT COUNT(*) as count FROM ${table} t
+           WHERE NOT EXISTS (
+             SELECT 1 FROM cloud_sync_state s
+             WHERE s.table_name = ? AND s.row_id = t.${pk}
+           )`
+      ).get(table);
+      untrackedCount += row?.count ?? 0;
+    }
+    return {
+      connected: this.config.enabled,
+      lastPushAt: pushCursor?.cursor_value ?? null,
+      lastPullAt: pullCursor?.cursor_value ?? null,
+      pendingPushCount: (pendingPush?.count ?? 0) + untrackedCount,
+      pendingPullCount: 0,
+      // Can't know without asking the server
+      conflictCount: conflicts?.count ?? 0,
+      endpoint: this.config.endpoint
+    };
+  }
+  // --- Internal: Table helpers ---
+  tableExists(name) {
+    const row = this.db.prepare(`SELECT 1 FROM sqlite_master WHERE type='table' AND name=?`).get(name);
+    return !!row;
+  }
+  ensureSyncTables() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS cloud_sync_state (
+        table_name TEXT NOT NULL,
+        row_id TEXT NOT NULL,
+        last_pushed_at INTEGER,
+        last_pushed_version INTEGER,
+        last_pulled_at INTEGER,
+        last_pulled_version INTEGER,
+        sync_status TEXT NOT NULL DEFAULT 'pending',
+        push_error TEXT,
+        push_attempts INTEGER NOT NULL DEFAULT 0,
+        PRIMARY KEY (table_name, row_id)
+      );
+      CREATE TABLE IF NOT EXISTS cloud_sync_cursors (
+        direction TEXT NOT NULL PRIMARY KEY,
+        cursor_value TEXT NOT NULL,
+        updated_at INTEGER NOT NULL
+      );
+    `);
+  }
+  // --- Internal: Collect pending entities ---
+  collectPendingEntities() {
+    this.ensureSyncTables();
+    const entities = [];
+    for (const table of SYNCABLE_TABLES) {
+      if (!this.tableExists(table)) continue;
+      const pk = PK_COLUMN[table];
+      const versionCol = VERSION_COLUMN[table];
+      const rows = this.db.prepare(
+        `SELECT t.* FROM ${table} t
+           WHERE NOT EXISTS (
+             SELECT 1 FROM cloud_sync_state s
+             WHERE s.table_name = ? AND s.row_id = t.${pk} AND s.sync_status = 'synced'
+           )
+           LIMIT ?`
+      ).all(table, this.config.batchSize * 2);
+      for (const row of rows) {
+        const id = String(row[pk]);
+        const version = Number(row[versionCol] ?? 0);
+        const tier = this.getRowTier(table, row);
+        const data = this.projectByTier(table, row, tier);
+        entities.push({ table, id, version, tier, data });
+      }
+    }
+    return entities;
+  }
+  // --- Internal: Generational projection ---
+  getRowTier(table, row) {
+    if (table !== "frames") return "young";
+    const createdAt = Number(row.created_at ?? 0);
+    const now = Math.floor(Date.now() / 1e3);
+    const ageDays = (now - createdAt) / 86400;
+    if (ageDays <= this.config.generationalPolicy.youngMaxAgeDays) {
+      return "young";
+    }
+    if (ageDays <= this.config.generationalPolicy.matureMaxAgeDays) {
+      return "mature";
+    }
+    return "old";
+  }
+  projectByTier(table, row, tier) {
+    if (table !== "frames") return row;
+    if (tier === "young") return row;
+    if (tier === "mature") {
+      return {
+        frame_id: row.frame_id,
+        run_id: row.run_id,
+        project_id: row.project_id,
+        name: row.name,
+        state: row.state,
+        digest_text: row.digest_text,
+        digest_json: row.digest_json,
+        importance_score: row.importance_score,
+        created_at: row.created_at,
+        closed_at: row.closed_at
+      };
+    }
+    return {
+      frame_id: row.frame_id,
+      project_id: row.project_id,
+      name: row.name,
+      state: row.state,
+      created_at: row.created_at,
+      closed_at: row.closed_at
+    };
+  }
+  // --- Internal: Apply pulled entities ---
+  applyPulledEntities(entities) {
+    let applied = 0;
+    let conflicts = 0;
+    const applyAll = this.db.transaction(() => {
+      for (const entity of entities) {
+        const pk = PK_COLUMN[entity.table];
+        const existing = this.db.prepare(
+          `SELECT ${pk} as id, ${VERSION_COLUMN[entity.table]} as version FROM ${entity.table} WHERE ${pk} = ?`
+        ).get(String(entity.data[pk]));
+        if (existing) {
+          if (entity.version > existing.version) {
+            this.upsertRow(entity.table, entity.data, pk);
+            this.markPulled(
+              entity.table,
+              String(entity.data[pk]),
+              entity.version
+            );
+            applied++;
+          } else {
+            conflicts++;
+            this.updateSyncState(
+              entity.table,
+              String(entity.data[pk]),
+              "conflict"
+            );
+          }
+        } else {
+          this.upsertRow(entity.table, entity.data, pk);
+          this.markPulled(
+            entity.table,
+            String(entity.data[pk]),
+            entity.version
+          );
+          applied++;
+        }
+      }
+    });
+    applyAll();
+    return { applied, conflicts };
+  }
+  upsertRow(table, data, pk) {
+    const valid = VALID_COLUMNS[table];
+    const safeEntries = Object.entries(data).filter(([k]) => valid.has(k));
+    if (safeEntries.length === 0) return;
+    const keys = safeEntries.map(([k]) => k);
+    const values = safeEntries.map(([, v]) => v);
+    const placeholders = keys.map(() => "?").join(", ");
+    const columns = keys.join(", ");
+    const updates = keys.filter((k) => k !== pk).map((k) => `${k} = excluded.${k}`).join(", ");
+    this.db.prepare(
+      `INSERT INTO ${table} (${columns}) VALUES (${placeholders})
+         ON CONFLICT(${pk}) DO UPDATE SET ${updates}`
+    ).run(...values);
+  }
+  // --- Internal: Sync state management ---
+  markPushed(table, rowId, version) {
+    const now = Date.now();
+    this.db.prepare(
+      `INSERT INTO cloud_sync_state (table_name, row_id, last_pushed_at, last_pushed_version, sync_status, push_attempts)
+         VALUES (?, ?, ?, ?, 'synced', 0)
+         ON CONFLICT(table_name, row_id) DO UPDATE SET
+           last_pushed_at = ?, last_pushed_version = ?, sync_status = 'synced', push_error = NULL, push_attempts = 0`
+    ).run(table, rowId, now, version, now, version);
+  }
+  markPulled(table, rowId, version) {
+    const now = Date.now();
+    this.db.prepare(
+      `INSERT INTO cloud_sync_state (table_name, row_id, last_pulled_at, last_pulled_version, sync_status)
+         VALUES (?, ?, ?, ?, 'synced')
+         ON CONFLICT(table_name, row_id) DO UPDATE SET
+           last_pulled_at = ?, last_pulled_version = ?, sync_status = 'synced'`
+    ).run(table, rowId, now, version, now, version);
+  }
+  updateSyncState(table, rowId, status) {
+    this.db.prepare(
+      `INSERT INTO cloud_sync_state (table_name, row_id, sync_status)
+         VALUES (?, ?, ?)
+         ON CONFLICT(table_name, row_id) DO UPDATE SET sync_status = ?`
+    ).run(table, rowId, status, status);
+  }
+  recordPushError(table, rowId, error) {
+    this.db.prepare(
+      `INSERT INTO cloud_sync_state (table_name, row_id, sync_status, push_error, push_attempts)
+         VALUES (?, ?, 'pending', ?, 1)
+         ON CONFLICT(table_name, row_id) DO UPDATE SET
+           push_error = ?, push_attempts = push_attempts + 1`
+    ).run(table, rowId, error, error);
+  }
+  // --- Internal: Cursor management ---
+  getCursor(direction) {
+    const row = this.db.prepare(
+      `SELECT cursor_value FROM cloud_sync_cursors WHERE direction = ?`
+    ).get(direction);
+    return row?.cursor_value ?? (/* @__PURE__ */ new Date(0)).toISOString();
+  }
+  setCursor(direction, cursor) {
+    this.db.prepare(
+      `INSERT INTO cloud_sync_cursors (direction, cursor_value, updated_at)
+         VALUES (?, ?, ?)
+         ON CONFLICT(direction) DO UPDATE SET cursor_value = ?, updated_at = ?`
+    ).run(direction, cursor, Date.now(), cursor, Date.now());
+  }
+  // --- Internal: HTTP ---
+  async sendPush(request) {
+    return this.httpPost("/v1/sync/push", request);
+  }
+  async sendPull(request) {
+    return this.httpPost("/v1/sync/pull", request);
+  }
+  async httpPost(path, body) {
+    let lastError;
+    for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
+      try {
+        const controller = new AbortController();
+        const timeout = setTimeout(
+          () => controller.abort(),
+          this.config.timeoutMs
+        );
+        try {
+          const response = await this.fetchFn(
+            `${this.config.endpoint}${path}`,
+            {
+              method: "POST",
+              headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.config.apiKey}`,
+                "X-Client-Id": this.config.clientId
+              },
+              body: JSON.stringify(body),
+              signal: controller.signal
+            }
+          );
+          if (!response.ok) {
+            lastError = `HTTP ${response.status}: ${response.statusText}`;
+            if (response.status >= 400 && response.status < 500) {
+              logger.error("Cloud sync client error", {
+                path,
+                status: response.status
+              });
+              return null;
+            }
+            continue;
+          }
+          return await response.json();
+        } finally {
+          clearTimeout(timeout);
+        }
+      } catch (error) {
+        lastError = error instanceof Error ? error.message : String(error);
+        if (attempt < this.config.retryAttempts - 1) {
+          const delay = this.config.retryBaseDelayMs * Math.pow(2, attempt);
+          await new Promise((r) => setTimeout(r, delay));
+        }
+      }
+    }
+    logger.warn("Cloud sync HTTP failed after retries", {
+      path,
+      error: lastError
+    });
+    return null;
+  }
+  // --- Internal: Checksum ---
+  computeChecksum(entities) {
+    const ids = entities.map((e) => `${e.table}:${e.id}`).sort().join(",");
+    return createHash("sha256").update(ids).digest("hex");
+  }
+}
+export {
+  CloudSyncEngine
+};

package/dist/src/core/storage/two-tier-storage.js

@@ -8,7 +8,11 @@ import {
   PutObjectCommand,
   GetObjectCommand
 } from "@aws-sdk/client-s3";
-import { createClient as createRedisClient } from "redis";
+let createRedisClient;
+try {
+  createRedisClient = require("redis").createClient;
+} catch {
+}
 import { Pool } from "pg";
 import * as zlib from "zlib";
 import { promisify } from "util";

package/dist/src/core/tasks/master-tasks-template.js

@@ -0,0 +1,43 @@
+import { fileURLToPath as __fileURLToPath } from 'url';
+import { dirname as __pathDirname } from 'path';
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+const MASTER_TASKS_TEMPLATE = `# Master Tasks
+
+> Single source of truth for what to build. Local-first, optionally syncs to Linear/GitHub.
+> Powers \\\`/next\\\` task selection. Referenced by CLAUDE.md and AGENTS.md.
+
+## Rules
+
+1. **Local-first**: This file is canonical. Linear/GH are downstream mirrors, not sources.
+2. **One owner**: Every task has exactly one owner. \\\`@me\\\` = you, \\\`@agent\\\` = dispatch to sub-agent, \\\`@defer\\\` = not assigned.
+3. **Priority tiers**: \\\`P0\\\` now (blocking), \\\`P1\\\` this week, \\\`P2\\\` next sprint, \\\`P3\\\` someday.
+4. **Status flow**: \\\`todo\\\` \u2192 \\\`active\\\` \u2192 \\\`done\\\` | \\\`blocked\\\` | \\\`cut\\\`.
+5. **Sync targets**: \\\`local\\\` (stays here), \\\`linear\\\` (create/update Linear issue), \\\`gh\\\` (GitHub issue/PR).
+6. **Agent /next reads P0 first, then P1**: Skip blocked, done, cut. Prefer @agent tasks unless @me explicitly set.
+7. **Keep it scannable**: One line per task in the table. Details go in notes column or linked doc.
+8. **Update on completion**: Mark done with date. Don't delete \u2014 move to Done section monthly.
+
+## Active Tasks
+
+| id | P | status | owner | sync | task | branch/PR | notes |
+|----|---|--------|-------|------|------|-----------|-------|
+
+## Done (archive monthly)
+
+_Move completed tasks here at end of month._
+
+<!-- Template row:
+| Txx | Px | todo | @me/@agent/@defer | local/linear/gh | description | branch/PR | notes |
+-->
+`;
+const TASKS_CONFIG_TEMPLATE = {
+  linear: { team: "", project: "" },
+  github: { repo: "" },
+  defaultSync: "local",
+  defaultOwner: "@me"
+};
+export {
+  MASTER_TASKS_TEMPLATE,
+  TASKS_CONFIG_TEMPLATE
+};