@threadbase-sh/streamer 1.61.1 → 1.62.0

package/README.md

@@ -160,6 +160,14 @@ Three layers: **core engine** (`src/*.ts`) → **API layer** (`src/api/` + `src/
 
 More detail: [docs/how-it-works.md](docs/how-it-works.md) and [docs/architecture/](docs/architecture/README.md).
 
+## Feature flags
+
+Streamer experiments are the `FEATURE_FLAGS` object in `src/feature-flags.ts`, keyed by name (`ptyHost`, `liveActivityPush`, …). `--feature <id=bool>` and `feature_flags:` in `server.yaml` use **those same camelCase keys**. Env vars are a different spelling (`THREADBASE_FEATURE_PTY_HOST`) and outrank both.
+
+Unknown yaml keys (typos, snake_case, the env name) are dropped with a warning; they do not stop the boot. Flags resolve once at boot — a change needs a restart. List and provenance: `GET /api/config/feature-flags`.
+
+Operator detail: [docs/guides/feature-flags.md](docs/guides/feature-flags.md). Implementation: `CLAUDE.md` § Feature flags.
+
 ## REST API
 
 Full endpoint reference: [docs/api-reference.md](docs/api-reference.md).

package/dist/cli.cjs

@@ -6146,8 +6146,15 @@ var init_logger = __esm({
 });
 
 // src/feature-flags.ts
+function isFeatureFlagId(id) {
+  return Object.hasOwn(FEATURE_FLAGS, id);
+}
+function getFeatureFlag(id) {
+  return { id, ...FEATURE_FLAGS[id] };
+}
 function findFeatureFlag(id) {
-  return FEATURE_FLAGS.find((f2) => f2.id === id);
+  if (!isFeatureFlagId(id)) return void 0;
+  return getFeatureFlag(id);
 }
 function parseBooleanEnv(raw2) {
   if (raw2 === void 0) return void 0;
@@ -6169,10 +6176,11 @@ function validateFeatureFlagValues(raw2) {
   }
   if (dropped.length > 0) {
     getLogger("feature-flags").warn(
-      `Ignoring unknown or non-boolean feature flags: ${dropped.join(", ")}`,
+      `Ignoring unknown or non-boolean feature flags: ${dropped.join(", ")}. Known ids (FEATURE_FLAGS keys, not env names): ${FEATURE_FLAG_IDS.join(", ")}`,
       {
         event: "config.feature_flags_dropped",
-        dropped
+        dropped,
+        known: FEATURE_FLAG_IDS
       }
     );
   }
@@ -6188,7 +6196,7 @@ function parseFeatureFlagArgs(entries) {
     const def = findFeatureFlag(id);
     if (!def) {
       errors.push(
-        `Unknown feature flag "${id}". Known flags: ${FEATURE_FLAGS.map((f2) => f2.id).join(", ")}`
+        `Unknown feature flag "${id}". Known flags (FEATURE_FLAGS keys, not env names): ${FEATURE_FLAG_IDS.join(", ")}`
       );
       continue;
     }
@@ -6204,7 +6212,7 @@ function resolveFeatureFlags(opts) {
   const env = opts?.env ?? process.env;
   const values = {};
   const sources = {};
-  for (const def of FEATURE_FLAGS) {
+  for (const def of FEATURE_FLAG_LIST) {
     const rungs = [
       ["override", opts?.override?.[def.id]],
       ["env", parseBooleanEnv(env[def.env])],
@@ -6218,50 +6226,50 @@ function resolveFeatureFlags(opts) {
   return { values, sources };
 }
 function nonDefaultFeatureFlags(values) {
-  return FEATURE_FLAGS.filter((f2) => values[f2.id] !== f2.default).map((f2) => f2.id);
+  return FEATURE_FLAG_LIST.filter((f2) => values[f2.id] !== f2.default).map((f2) => f2.id);
 }
 function describeFeatureFlags(resolution) {
-  return FEATURE_FLAGS.map(
+  return FEATURE_FLAG_LIST.map(
     (f2) => `${f2.id}=${resolution.values[f2.id]}(${resolution.sources[f2.id]})`
   ).join(" ");
 }
-var FEATURE_FLAGS;
+var FEATURE_FLAGS, FEATURE_FLAG_IDS, FEATURE_FLAG_LIST;
 var init_feature_flags = __esm({
   "src/feature-flags.ts"() {
     "use strict";
     init_logger();
-    FEATURE_FLAGS = [
-      {
-        id: "codexSystemPrompt",
+    FEATURE_FLAGS = {
+      codexSystemPrompt: {
         description: "Send the built system prompt to fresh Codex sessions. Off by default: Codex has no --system-prompt flag, so the prompt goes in the positional [PROMPT] argument, which Codex treats as the user's opening turn rather than a system-level instruction.",
         default: false,
         env: "THREADBASE_FEATURE_CODEX_SYSTEM_PROMPT"
       },
-      {
-        id: "sessionRehydration",
+      sessionRehydration: {
         description: "Seed the session list at boot with sessions a previous streamer run left behind, so a restart leaves them one tap from resuming instead of silently gone. On by default, with a kill switch: it changes what GET /api/sessions contains.",
         default: true,
         env: "THREADBASE_FEATURE_SESSION_REHYDRATION"
       },
-      {
-        id: "liveActivityPush",
+      liveActivityPush: {
         description: "Drive iOS Live Activity surfaces for running sessions. Off by default: the streamer half needs an APNs p8 and a registered push-to-start token, and without both, mobile falls back to starting the activity locally \u2014 which freezes the moment the app backgrounds and expires silently after ~8h. Mobile reads this flag and skips its local path too, so one switch turns the whole surface off.",
         default: false,
         env: "THREADBASE_FEATURE_LIVE_ACTIVITY_PUSH"
       },
-      {
-        id: "e2ee",
+      e2ee: {
         description: "Application-layer encryption between a paired device and this server, independent of TLS, so a tunnel or a LAN observer on the path carries ciphertext. Off by default: it is negotiated per device, never forced, because released mobile builds cannot be force-updated and a server that demanded it would break every one of them.",
         default: false,
         env: "THREADBASE_FEATURE_E2EE"
       },
-      {
-        id: "ptyHost",
+      ptyHost: {
         description: "Keep live PTYs in a separate host process so a streamer restart can reconnect without restarting the agents. Off by default until cross-platform behavior is qualified.",
         default: false,
         env: "THREADBASE_FEATURE_PTY_HOST"
       }
-    ];
+    };
+    FEATURE_FLAG_IDS = Object.keys(FEATURE_FLAGS);
+    FEATURE_FLAG_LIST = FEATURE_FLAG_IDS.map((id) => ({
+      id,
+      ...FEATURE_FLAGS[id]
+    }));
   }
 });
 
@@ -142534,7 +142542,13 @@ function openDatabase(dbPath) {
   if (dbPath !== ":memory:") {
     (0, import_fs15.mkdirSync)((0, import_path12.dirname)(dbPath), { recursive: true });
   }
-  const db = new import_better_sqlite3.default(dbPath);
+  let db;
+  try {
+    db = new import_better_sqlite3.default(dbPath);
+  } catch (err) {
+    if (isNativeBindingFailure(err)) throw nativeBindingError(err, dbPath);
+    throw err;
+  }
   db.pragma("journal_mode = WAL");
   db.pragma("synchronous = NORMAL");
   db.pragma("temp_store = MEMORY");
@@ -142544,6 +142558,30 @@ function openDatabase(dbPath) {
   getLogger2().debug({ dbPath }, "db: opened");
   return db;
 }
+function isNativeBindingFailure(err) {
+  const message = err instanceof Error ? err.message : String(err);
+  return /could not locate the bindings file|NODE_MODULE_VERSION|was compiled against|\.node['"\s]/i.test(
+    message
+  );
+}
+function nativeBindingError(err, dbPath) {
+  const detail = err instanceof Error ? err.message : String(err);
+  return new Error(
+    [
+      `Could not load better-sqlite3's native binary, so the persistent index at ${dbPath} cannot be opened.`,
+      "This usually means the binary was never built or downloaded \u2014 not that your Node version is wrong.",
+      "",
+      "Try, in order:",
+      "  1. npm rebuild better-sqlite3",
+      "  2. rm -rf node_modules && npm install",
+      "     (npm 12 blocks package install scripts by default; approve better-sqlite3 if asked)",
+      "  3. Run without SQLite entirely: new ConversationScanner({ persistent: false })",
+      "     or pass --no-persist on the CLI. Search falls back to the in-memory index.",
+      "",
+      `Original error: ${detail}`
+    ].join("\n")
+  );
+}
 var FULL_RECONCILE_EVERY_N_SCANS = 20;
 async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
   const log15 = getLogger2();
@@ -146911,7 +146949,10 @@ var ConversationHandlers = class {
       score: r.score,
       matches: Array.isArray(r.matches) ? r.matches.map((m2) => ({
         field: m2.field,
-        snippet: m2.snippet
+        snippet: m2.snippet,
+        // Offsets into `snippet` for the matched tokens. Absent on metadata
+        // hits, and on scanners older than the one that added them.
+        highlights: m2.highlights
       })) : []
     }));
     const page = paginate(applyFilters(adapted, filters), offset, limit);
@@ -148616,7 +148657,9 @@ var SessionHandlers = class {
       res.end(JSON.stringify({ error: "Session not found" }));
       return;
     }
+    const shouldForget = this.shouldForgetEmptySession(session);
     if (session.status === "idle") {
+      if (shouldForget) this.forgetEmptyStoppedSession(sessionId);
       res.writeHead(200, { "Content-Type": "application/json" });
       res.end(JSON.stringify({ status: "already_idle", sessionId }));
       return;
@@ -148644,6 +148687,7 @@ var SessionHandlers = class {
     this.ptyManager.putOnHold(sessionId);
     this.discoveryCache = null;
     const outcome = await Promise.race([idlePromise, timeoutPromise]);
+    if (shouldForget) this.forgetEmptyStoppedSession(sessionId);
     if (outcome === "idle") {
       res.write(`${JSON.stringify({ event: "stopped", sessionId })}
 `);
@@ -148656,6 +148700,42 @@ var SessionHandlers = class {
     }
     res.end();
   }
+  /**
+   * An unused start: the user never submitted a prompt, and the conversation
+   * cache has no row for this id (empty Codex/Claude often never write a JSONL).
+   * `conversationId === sessionId` is not evidence of history — only the cache
+   * is. promptCount > 0 or a cache hit keeps today's hold path.
+   */
+  shouldForgetEmptySession(session) {
+    const stored = this.sessionStore.getManaged(session.id);
+    const promptCount = Math.max(session.promptCount, stored?.promptCount ?? 0);
+    if (promptCount > 0) return false;
+    return !this.hasCachedConversationFor(session, stored);
+  }
+  hasCachedConversationFor(session, stored) {
+    const cache = this.cache;
+    if (!cache) return false;
+    const ids = /* @__PURE__ */ new Set([session.id]);
+    if (session.boundConversationId) ids.add(session.boundConversationId);
+    if (session.resumedFromConversationId) ids.add(session.resumedFromConversationId);
+    if (stored?.boundConversationId) ids.add(stored.boundConversationId);
+    if (stored?.resumedFromConversationId) ids.add(stored.resumedFromConversationId);
+    for (const id of ids) {
+      if (cache.hasConversation(id)) return true;
+    }
+    return false;
+  }
+  forgetEmptyStoppedSession(sessionId) {
+    this.log.info(`[stop] forgetting empty session ${sessionId.slice(0, 8)}`, {
+      event: "session.forget_empty",
+      sessionId
+    });
+    this.deps.forgetSession(sessionId);
+    this.wsHub.broadcast({
+      type: "session_list",
+      sessions: this.sessionStore.list(this.deps.ptyAttachedIds())
+    });
+  }
   async handleAdopt(sessionId, res) {
     const discovered = await discoverClaudeProcesses();
     this.sessionStore.setDiscovered(discovered);
@@ -154081,6 +154161,7 @@ var StreamerServer = class {
       spawnFlagOverrides: () => this.spawnFlagOverrides(),
       resolveConversationTarget: (sessionId) => this.resolveConversationTarget(sessionId),
       waitForStartupOutcome: (sessionId, timeoutMs) => this.waitForStartupOutcome(sessionId, timeoutMs),
+      forgetSession: (sessionId) => this.forgetSession(sessionId),
       abandonFailedStart: (sessionId) => this.abandonFailedStart(sessionId),
       enrichResumedSessionAsync: (sessionId, projectPath, conv) => this.enrichResumedSessionAsync(sessionId, projectPath, conv),
       findJsonlPath: (uuid3) => this.conversationHandlers.findJsonlPath(uuid3),
@@ -155072,7 +155153,7 @@ var StreamerServer = class {
    */
   getFeatureFlagsConfig() {
     return {
-      registry: FEATURE_FLAGS,
+      registry: FEATURE_FLAG_LIST,
       values: this.featureFlags,
       sources: this.featureFlagSources
     };
@@ -155306,29 +155387,44 @@ var StreamerServer = class {
     });
   }
   /**
-   * Drop every trace of a session that never became usable, and hand back what
-   * it failed with.
+   * Drop every trace of a managed session: in-memory store, durable registry
+   * row, and the collision-probe markers that would otherwise outlive it.
    *
-   * The runner has already torn itself down (failStartup / handleExit); what
-   * remains is server-side bookkeeping that would otherwise leave a dead
-   * session in the list, a registry row claiming a spawn, and a `selfPtyEndedAt`
-   * marker that would suppress the mtime collision signal on the NEXT resume —
-   * i.e. it would help hide the very owner we just collided with.
+   * Used when a start never became usable (`abandonFailedStart`) and when stop
+   * is asked to discard an empty session that has no cached conversation. The
+   * registry delete is load-bearing — `rehydrateSessions` will bring the row
+   * back on the next boot if it remains.
+   *
+   * Callers that kill the PTY (`putOnHold`) must do that *first*: onStatusChange
+   * on idle writes `selfPtyEndedAt` and a registry status, and those have to
+   * be cleared here afterwards.
    */
-  abandonFailedStart(sessionId) {
+  forgetSession(sessionId) {
     this.sessionStore.removeManaged(sessionId);
     this.selfPtyEndedAt.delete(sessionId);
     this.contendedSessions.delete(sessionId);
     try {
       this.managedSessionsRepo?.delete(sessionId);
     } catch (err) {
-      this.log.warn("[registry] failed to drop a failed start", {
+      this.log.warn("[registry] failed to drop a session", {
         event: "registry.forget_failed",
         sessionId,
         err
       });
     }
   }
+  /**
+   * Drop every trace of a session that never became usable.
+   *
+   * The runner has already torn itself down (failStartup / handleExit); what
+   * remains is server-side bookkeeping that would otherwise leave a dead
+   * session in the list, a registry row claiming a spawn, and a `selfPtyEndedAt`
+   * marker that would suppress the mtime collision signal on the NEXT resume —
+   * i.e. it would help hide the very owner we just collided with.
+   */
+  abandonFailedStart(sessionId) {
+    this.forgetSession(sessionId);
+  }
   enrichResumedSessionAsync(sessionId, projectPath, conv) {
     try {
       if (!this.sessionStore.getManaged(sessionId)) return;
@@ -159524,7 +159620,7 @@ program2.command("serve").description("Start the streamer server").option("-p, -
   (value, previous = []) => [...previous, value]
 ).option(
   "--feature <id=bool>",
-  "Enable or disable a server feature flag, e.g. --feature codexSystemPrompt=true. Repeatable. Overridden by the flag's THREADBASE_FEATURE_* env var; overrides feature_flags: in ~/.threadbase/server.yaml.",
+  `Enable or disable a server feature flag. Ids are the FEATURE_FLAGS keys (${FEATURE_FLAG_IDS.join(", ")}), e.g. --feature ptyHost=true \u2014 not the THREADBASE_FEATURE_* env names. Repeatable. Overridden by the flag's env var; overrides feature_flags: in ~/.threadbase/server.yaml.`,
   (value, previous = []) => [...previous, value]
 ).option(
   "--claude-extra-args <args>",