@vectorize-io/hindsight-coding-agents 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,9 +7,25 @@ import { existsSync, readFileSync as readFileSync3 } from "fs";
7
7
  import { execFileSync } from "child_process";
8
8
  import { homedir } from "os";
9
9
  import { basename, dirname, join, normalize, sep } from "path";
10
+
11
+ // src/core/template.ts
12
+ var PLACEHOLDER = /\{([a-zA-Z_]+)\}/g;
13
+ function applyTemplate(template, resolvers, what) {
14
+ return template.replace(PLACEHOLDER, (_, name) => {
15
+ const resolve2 = resolvers[name];
16
+ if (!resolve2) {
17
+ console.error(
18
+ `hindsight: unknown ${what} placeholder "{${name}}" \u2014 valid: ` + Object.keys(resolvers).sort().map((k) => `{${k}}`).join(", ")
19
+ );
20
+ return "unknown";
21
+ }
22
+ return resolve2();
23
+ });
24
+ }
25
+
26
+ // src/core/bank.ts
10
27
  var DEFAULT_BANK_NAME = "coding";
11
28
  var DEFAULT_TEMPLATE = "coding-agent::{gitProject}";
12
- var PLACEHOLDER = /\{([a-zA-Z]+)\}/g;
13
29
  function getProjectRootFromGit(directory) {
14
30
  if (!directory) return null;
15
31
  try {
@@ -24,6 +40,9 @@ function getProjectRootFromGit(directory) {
24
40
  return null;
25
41
  }
26
42
  }
43
+ function projectNameOf(directory) {
44
+ return gitProjectName(directory, true);
45
+ }
27
46
  function gitProjectName(directory, resolveWorktrees) {
28
47
  if (resolveWorktrees) {
29
48
  const root = getProjectRootFromGit(directory);
@@ -55,16 +74,7 @@ function deriveBankId(config, directory, harness = "coding") {
55
74
  channel: () => process.env.HINDSIGHT_CHANNEL_ID || "default",
56
75
  user: () => process.env.HINDSIGHT_USER_ID || "anonymous"
57
76
  };
58
- return (config.bankIdTemplate || DEFAULT_TEMPLATE).replace(PLACEHOLDER, (_, name) => {
59
- const r = resolvers[name];
60
- if (!r) {
61
- console.error(
62
- `hindsight: unknown bankIdTemplate placeholder "{${name}}" \u2014 valid: ` + Object.keys(resolvers).sort().map((k) => `{${k}}`).join(", ")
63
- );
64
- return "unknown";
65
- }
66
- return r();
67
- });
77
+ return applyTemplate(config.bankIdTemplate || DEFAULT_TEMPLATE, resolvers, "bankIdTemplate");
68
78
  }
69
79
 
70
80
  // src/core/config.ts
@@ -118,6 +128,12 @@ function resolveConfig(raw = {}) {
118
128
  surveyBudgetUsd: raw.surveyBudgetUsd || 2,
119
129
  surveyRefreshCommits: raw.surveyRefreshCommits ?? 20,
120
130
  gitIngest: ["message", "full", "none"].includes(raw.gitIngest) ? raw.gitIngest : "message",
131
+ // Hostile input is a config typo, not an attack: keep only string entries so a stray number or
132
+ // nested object cannot reach the API as a tag and fail the whole retain.
133
+ retainTags: Array.isArray(raw.retainTags) ? raw.retainTags.filter((t) => typeof t === "string" && t.trim() !== "") : [],
134
+ retainMetadata: raw.retainMetadata && typeof raw.retainMetadata === "object" ? Object.fromEntries(
135
+ Object.entries(raw.retainMetadata).filter(([, v]) => typeof v === "string")
136
+ ) : {},
121
137
  banks: raw.banks && typeof raw.banks === "object" ? raw.banks : {},
122
138
  logLevel: ["debug", "info", "warn", "error"].includes(raw.logLevel) ? raw.logLevel : "info"
123
139
  };
@@ -171,7 +187,10 @@ var ENV_KEYS = {
171
187
  surveyBudgetUsd: "HINDSIGHT_SURVEY_BUDGET_USD",
172
188
  surveyRefreshCommits: "HINDSIGHT_SURVEY_REFRESH_COMMITS",
173
189
  logLevel: "HINDSIGHT_LOG_LEVEL",
174
- gitIngest: "HINDSIGHT_GIT_INGEST"
190
+ gitIngest: "HINDSIGHT_GIT_INGEST",
191
+ // Comma-separated, e.g. HINDSIGHT_RETAIN_TAGS="project:{gitProject},env:work". A LIST rather than
192
+ // a map, so it flattens cleanly; its sibling retainMetadata stays file-only for the reason above.
193
+ retainTags: "HINDSIGHT_RETAIN_TAGS"
175
194
  };
176
195
  var ENV_BOOLEANS = /* @__PURE__ */ new Set([
177
196
  "dynamicBankId",
@@ -182,6 +201,7 @@ var ENV_BOOLEANS = /* @__PURE__ */ new Set([
182
201
  "autoSeed",
183
202
  "codebaseSurvey"
184
203
  ]);
204
+ var ENV_LISTS = /* @__PURE__ */ new Set(["retainTags"]);
185
205
  var ENV_NUMBERS = /* @__PURE__ */ new Set([
186
206
  "apiPort",
187
207
  "daemonIdleTimeout",
@@ -199,6 +219,9 @@ function readEnvConfig(env = process.env) {
199
219
  if (value === void 0 || value.trim() === "") continue;
200
220
  if (ENV_BOOLEANS.has(key)) {
201
221
  out[key] = ["1", "true", "yes", "on"].includes(value.trim().toLowerCase());
222
+ } else if (ENV_LISTS.has(key)) {
223
+ const items = value.split(",").map((v) => v.trim()).filter(Boolean);
224
+ if (items.length) out[key] = items;
202
225
  } else if (ENV_NUMBERS.has(key)) {
203
226
  const n = Number(value);
204
227
  if (Number.isFinite(n)) out[key] = n;
@@ -413,11 +436,32 @@ var CODING_BANK_TEMPLATE = {
413
436
  entities_allow_free_form: true
414
437
  }
415
438
  };
439
+ var CODING_BANK_STRUCTURE = {
440
+ version: "1",
441
+ bank: {
442
+ retain_strategies: RETAIN_STRATEGIES,
443
+ entity_labels: [KNOWLEDGE_LABELS]
444
+ }
445
+ };
416
446
 
417
447
  // src/core/util.ts
418
448
  var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
449
+ function semverGte(version, min) {
450
+ const parts = (v) => {
451
+ const m = v.trim().match(/^(\d+)\.(\d+)(?:\.(\d+))?/);
452
+ return m ? [Number(m[1]), Number(m[2]), Number(m[3] ?? 0)] : void 0;
453
+ };
454
+ const a = version ? parts(version) : void 0;
455
+ const b = parts(min);
456
+ if (!a || !b) return false;
457
+ for (let i = 0; i < 3; i++) {
458
+ if (a[i] !== b[i]) return a[i] > b[i];
459
+ }
460
+ return true;
461
+ }
419
462
 
420
463
  // src/core/hindsight.ts
464
+ var MIN_IDEMPOTENT_RETAIN_VERSION = "0.8.6";
421
465
  var KnowledgePagesUnavailableError = class extends Error {
422
466
  code = "knowledge_pages_unavailable";
423
467
  constructor() {
@@ -426,6 +470,7 @@ var KnowledgePagesUnavailableError = class extends Error {
426
470
  }
427
471
  };
428
472
  var TERMINAL = /* @__PURE__ */ new Set(["completed", "failed", "cancelled", "error"]);
473
+ var MISSION_FIELDS = ["reflect_mission", "retain_mission", "observations_mission"];
429
474
  var HindsightClient = class {
430
475
  apiUrl;
431
476
  apiToken;
@@ -434,6 +479,8 @@ var HindsightClient = class {
434
479
  // async operation ids collected by retain(), for drain()
435
480
  /** Tri-state capability probe: unknown until the first page request, then cached. */
436
481
  knowledgePagesSupported;
482
+ /** Tri-state capability probe: unknown until the first append-mode retain, then cached. */
483
+ idempotentRetain;
437
484
  log;
438
485
  constructor(o) {
439
486
  this.apiUrl = o.apiUrl.replace(/\/$/, "");
@@ -462,7 +509,8 @@ var HindsightClient = class {
462
509
  throw new Error(`${method} ${url} -> ${r.status} ${await r.text()}`);
463
510
  return r;
464
511
  }
465
- /** Retain one memory. Async by default: enqueue extraction server-side and collect its op-id for drain(). */
512
+ /** Retain one memory. ALWAYS async: enqueue extraction server-side and collect its op-id for
513
+ * drain(). Nothing in this plugin can afford to block a coding agent's hook on extraction. */
466
514
  async retain(content, context, documentId, tags, strategy, opts = {}) {
467
515
  const item = {
468
516
  content,
@@ -473,14 +521,36 @@ var HindsightClient = class {
473
521
  };
474
522
  if (opts.timestamp) item.timestamp = opts.timestamp;
475
523
  if (opts.metadata) item.metadata = opts.metadata;
476
- const isAsync = opts.async !== false;
477
- const r = await this.req("POST", this.bankUrl("/memories"), { items: [item], async: isAsync });
478
- if (isAsync) {
479
- try {
480
- const j = await r.json();
481
- if (j.operation_id) this.opIds.push(j.operation_id);
482
- } catch {
483
- }
524
+ if (opts.updateMode) item.update_mode = opts.updateMode;
525
+ const body = { items: [item], async: true };
526
+ if (opts.operationId) body.operation_id = opts.operationId;
527
+ const r = await this.req("POST", this.bankUrl("/memories"), body);
528
+ try {
529
+ const j = await r.json();
530
+ if (j.operation_id) this.opIds.push(j.operation_id);
531
+ } catch {
532
+ }
533
+ }
534
+ /**
535
+ * Whether this server honours `operation_id` on an async retain, and can therefore be appended to
536
+ * safely (see MIN_IDEMPOTENT_RETAIN_VERSION). Probed once per client via GET /version; anything
537
+ * unreachable, unparseable or older answers "no", which costs efficiency, never correctness.
538
+ */
539
+ async supportsIdempotentRetain() {
540
+ if (this.idempotentRetain === void 0) {
541
+ this.idempotentRetain = await this.probeIdempotentRetain();
542
+ this.log(`server retain idempotency: ${this.idempotentRetain ? "supported" : "unavailable"}`);
543
+ }
544
+ return this.idempotentRetain;
545
+ }
546
+ async probeIdempotentRetain() {
547
+ try {
548
+ const r = await this.req("GET", `${this.apiUrl}/version`);
549
+ if (!r.ok) return false;
550
+ const j = await r.json();
551
+ return semverGte(j.api_version, MIN_IDEMPOTENT_RETAIN_VERSION);
552
+ } catch {
553
+ return false;
484
554
  }
485
555
  }
486
556
  /**
@@ -517,12 +587,39 @@ var HindsightClient = class {
517
587
  await this.req("DELETE", this.bankUrl());
518
588
  this.log(`[bank] reset ${this.bank}`);
519
589
  }
520
- await this.req("POST", this.bankUrl("/import"), CODING_BANK_TEMPLATE);
590
+ const seeded = opts.reset ? false : await this.hasBankMissions();
591
+ await this.req(
592
+ "POST",
593
+ this.bankUrl("/import"),
594
+ seeded ? CODING_BANK_STRUCTURE : CODING_BANK_TEMPLATE
595
+ );
521
596
  this.log(
522
- `[bank] template applied to ${this.bank}: missions, entity_labels {knowledge}, strategies {git, gitlog, conversation, document}`
597
+ seeded ? `[bank] structure applied to ${this.bank}: entity_labels {knowledge}, strategies {git, gitlog, conversation, document} \u2014 missions left as configured` : `[bank] template applied to ${this.bank}: missions, entity_labels {knowledge}, strategies {git, gitlog, conversation, document}`
523
598
  );
524
599
  await this.seedPages();
525
600
  }
601
+ /**
602
+ * Whether this bank already carries bank-level mission overrides — ours from an earlier seed, or
603
+ * the user's own edit. Either way they are not ours to overwrite.
604
+ *
605
+ * Reads the bank-scoped OVERRIDES, not the resolved config, so inherited global defaults don't
606
+ * read as "already set". A missing bank, or a deployment with the bank-config API switched off,
607
+ * answers false: there is nothing to preserve, and without that API a user cannot set per-bank
608
+ * missions in the first place.
609
+ */
610
+ async hasBankMissions() {
611
+ try {
612
+ const r = await this.req("GET", this.bankUrl("/config"));
613
+ if (!r.ok) return false;
614
+ const j = await r.json();
615
+ return MISSION_FIELDS.some((f) => {
616
+ const v = j.overrides?.[f];
617
+ return typeof v === "string" && v.trim() !== "";
618
+ });
619
+ } catch {
620
+ return false;
621
+ }
622
+ }
526
623
  /** Delete one document (cascades its memory units/links). Used by deepen's self-cleanup. */
527
624
  async deleteDocument(documentId) {
528
625
  await this.req("DELETE", this.bankUrl(`/documents/${encodeURIComponent(documentId)}`));
@@ -768,8 +865,7 @@ var HindsightClient = class {
768
865
  "initiative marker",
769
866
  markerId,
770
867
  ["knowledge:feature-work", `relatedPageId:${pageId}`],
771
- "document",
772
- { async: true }
868
+ "document"
773
869
  );
774
870
  return { page_id: pageId };
775
871
  }
@@ -796,9 +892,73 @@ var HindsightClient = class {
796
892
  import { mkdirSync as mkdirSync2, readFileSync as readFileSync2, writeFileSync } from "fs";
797
893
  import { tmpdir as tmpdir2 } from "os";
798
894
  import { dirname as dirname4, join as join5 } from "path";
895
+ function sessionCacheFile(harness, sessionId) {
896
+ return join5(tmpdir2(), `hindsight-${harness}`, `${sessionId}.json`);
897
+ }
898
+ function readSessionCache(cacheFile) {
899
+ try {
900
+ return JSON.parse(readFileSync2(cacheFile, "utf8"));
901
+ } catch {
902
+ return {};
903
+ }
904
+ }
905
+ function writeSessionCache(cacheFile, cache) {
906
+ try {
907
+ mkdirSync2(dirname4(cacheFile), { recursive: true });
908
+ writeFileSync(cacheFile, JSON.stringify(cache));
909
+ } catch {
910
+ }
911
+ }
912
+ function fileCursorStore(harness) {
913
+ return {
914
+ read: (sessionId) => readSessionCache(sessionCacheFile(harness, sessionId)).retain,
915
+ write: (sessionId, cursor) => {
916
+ const file = sessionCacheFile(harness, sessionId);
917
+ writeSessionCache(file, { ...readSessionCache(file), retain: cursor });
918
+ }
919
+ };
920
+ }
799
921
 
800
922
  // src/core/retain-hook.ts
801
- import { readFileSync as readFileSync5 } from "fs";
923
+ import { readFileSync as readFileSync4 } from "fs";
924
+
925
+ // src/core/retain-cursor.ts
926
+ import { createHash } from "crypto";
927
+ function fingerprintTurns(turns, count) {
928
+ const h = createHash("sha1").update(String(count));
929
+ for (let i = 0; i < count; i++) h.update("\n" + JSON.stringify(turns[i]));
930
+ return h.digest("hex");
931
+ }
932
+ function planRetain(turns, cursor, opts) {
933
+ if (!turns.length) return { mode: "skip" };
934
+ if (!opts.appendSupported || !cursor || cursor.dirty) return { mode: "replace" };
935
+ if (cursor.bank !== opts.bank) return { mode: "replace" };
936
+ if (cursor.turns > turns.length) return { mode: "replace" };
937
+ if (fingerprintTurns(turns, cursor.turns) !== cursor.fingerprint) return { mode: "replace" };
938
+ if (cursor.turns === turns.length) return { mode: "skip" };
939
+ return { mode: "append", fromTurn: cursor.turns };
940
+ }
941
+
942
+ // src/core/uuid.ts
943
+ import { createHash as createHash2 } from "crypto";
944
+ var HINDSIGHT_NAMESPACE = "e75686a4-e923-4326-a0d1-358f1c6c3eb4";
945
+ function uuidV5(name, namespace = HINDSIGHT_NAMESPACE) {
946
+ const ns = Buffer.from(namespace.replace(/-/g, ""), "hex");
947
+ if (ns.length !== 16) throw new Error(`invalid UUID namespace: ${namespace}`);
948
+ const bytes = Buffer.from(
949
+ createHash2("sha1").update(ns).update(Buffer.from(name, "utf8")).digest().subarray(0, 16)
950
+ );
951
+ bytes[6] = bytes[6] & 15 | 80;
952
+ bytes[8] = bytes[8] & 63 | 128;
953
+ const hex = bytes.toString("hex");
954
+ return [
955
+ hex.slice(0, 8),
956
+ hex.slice(8, 12),
957
+ hex.slice(12, 16),
958
+ hex.slice(16, 20),
959
+ hex.slice(20)
960
+ ].join("-");
961
+ }
802
962
 
803
963
  // src/core/chat.ts
804
964
  function withRefId(refId, turns, baseTs) {
@@ -807,18 +967,73 @@ function withRefId(refId, turns, baseTs) {
807
967
  function renderSessionJsonl(refId, turns, baseTs) {
808
968
  return withRefId(refId, turns, baseTs).map((t) => JSON.stringify(t)).join("\n");
809
969
  }
810
- async function retainLiveSession(client, sessionId, turns, startTs, harness) {
970
+ async function supportsAppend(client) {
971
+ try {
972
+ return await client.supportsIdempotentRetain();
973
+ } catch {
974
+ return false;
975
+ }
976
+ }
977
+ var writeBacks = /* @__PURE__ */ new WeakMap();
978
+ function serialize(cursors, sessionId, write2) {
979
+ const perSession = writeBacks.get(cursors) ?? /* @__PURE__ */ new Map();
980
+ writeBacks.set(cursors, perSession);
981
+ const next = (perSession.get(sessionId) ?? Promise.resolve()).then(write2, write2);
982
+ const tail = next.catch(() => {
983
+ });
984
+ perSession.set(sessionId, tail);
985
+ void tail.then(() => {
986
+ if (perSession.get(sessionId) === tail) perSession.delete(sessionId);
987
+ });
988
+ return next;
989
+ }
990
+ async function retainLiveSession(client, sessionId, turns, startTs, harness, opts = {}) {
991
+ const cursors = opts.cursors;
992
+ if (!cursors) return writeSession(client, sessionId, turns, startTs, harness, opts.stamp);
993
+ return serialize(
994
+ cursors,
995
+ sessionId,
996
+ () => writeSession(client, sessionId, turns, startTs, harness, opts.stamp, cursors)
997
+ );
998
+ }
999
+ async function writeSession(client, sessionId, turns, startTs, harness, stamp, cursors) {
811
1000
  const refId = `conversation:${sessionId}`;
1001
+ const appendSupported = Boolean(cursors) && await supportsAppend(client);
1002
+ const plan = planRetain(turns, cursors?.read(sessionId), { appendSupported, bank: client.bank });
1003
+ if (plan.mode === "skip") return;
1004
+ const content = plan.mode === "append" ? turns.slice(plan.fromTurn).map((t) => JSON.stringify(t)).join("\n") : renderSessionJsonl(refId, turns, startTs);
1005
+ const next = {
1006
+ turns: turns.length,
1007
+ fingerprint: fingerprintTurns(turns, turns.length),
1008
+ bank: client.bank
1009
+ };
1010
+ cursors?.write(sessionId, { ...next, dirty: true });
812
1011
  await client.retain(
813
- renderSessionJsonl(refId, turns, startTs),
1012
+ content,
814
1013
  "coding agent session",
815
1014
  refId,
816
- ["source:chat", ...harness ? [`harness:${harness}`] : []],
1015
+ // Configured tags first, built-ins last and deduped: `source:chat` and `harness:<id>` are what
1016
+ // the documents list filters and draws its agent logo from, so a template cannot displace them.
1017
+ [
1018
+ .../* @__PURE__ */ new Set([
1019
+ ...stamp?.tags ?? [],
1020
+ "source:chat",
1021
+ ...harness ? [`harness:${harness}`] : []
1022
+ ])
1023
+ ],
817
1024
  "conversation",
818
1025
  {
819
1026
  timestamp: startTs,
820
- async: true,
1027
+ updateMode: plan.mode === "append" ? "append" : void 0,
1028
+ // Identity of THIS payload: a resubmission of the same bytes is collapsed server-side into
1029
+ // the original operation instead of extracting (or appending) twice.
1030
+ operationId: uuidV5(`${client.bank}
1031
+ ${refId}
1032
+ ${plan.mode}
1033
+ ${content}`),
821
1034
  metadata: {
1035
+ ...stamp?.metadata,
1036
+ // configured first: the built-ins below win on any key collision
822
1037
  source: "chat",
823
1038
  session_id: sessionId,
824
1039
  ref_id: refId,
@@ -826,6 +1041,7 @@ async function retainLiveSession(client, sessionId, turns, startTs, harness) {
826
1041
  }
827
1042
  }
828
1043
  );
1044
+ cursors?.write(sessionId, next);
829
1045
  }
830
1046
 
831
1047
  // src/core/daemon.ts
@@ -944,11 +1160,116 @@ function preflightDaemon(cfg, harness) {
944
1160
  return true;
945
1161
  }
946
1162
 
947
- // src/core/transcript.ts
948
- import { readFileSync as readFileSync4 } from "fs";
1163
+ // src/core/retain-stamp.ts
1164
+ import { basename as basename2 } from "path";
1165
+ var RESERVED_TAG_PREFIX = /^(source|harness):/;
1166
+ function resolversFor(ctx) {
1167
+ return {
1168
+ // Worktree-aware like the bank id, so every linked worktree of a repo stamps the SAME name —
1169
+ // otherwise a shared bank ends up with `project:app` and `project:app-wt2` for one repository.
1170
+ gitProject: () => projectNameOf(ctx.directory),
1171
+ project: () => ctx.directory ? basename2(ctx.directory) : "unknown",
1172
+ harness: () => ctx.harness,
1173
+ bankId: () => ctx.bankId,
1174
+ sessionId: () => ctx.sessionId,
1175
+ timestamp: () => (/* @__PURE__ */ new Date()).toISOString(),
1176
+ channel: () => process.env.HINDSIGHT_CHANNEL_ID || "default",
1177
+ user: () => process.env.HINDSIGHT_USER_ID || "anonymous"
1178
+ };
1179
+ }
1180
+ function buildRetainStamp(cfg, ctx) {
1181
+ const hasTags = Boolean(cfg.retainTags?.length);
1182
+ const hasMetadata = Boolean(cfg.retainMetadata && Object.keys(cfg.retainMetadata).length);
1183
+ if (!hasTags && !hasMetadata) return { tags: [], metadata: {} };
1184
+ const base = resolversFor(ctx);
1185
+ const cache = /* @__PURE__ */ new Map();
1186
+ const resolvers = Object.fromEntries(
1187
+ Object.entries(base).map(([name, resolve2]) => [
1188
+ name,
1189
+ () => {
1190
+ const hit = cache.get(name);
1191
+ if (hit !== void 0) return hit;
1192
+ const value = resolve2();
1193
+ cache.set(name, value);
1194
+ return value;
1195
+ }
1196
+ ])
1197
+ );
1198
+ const tags = (cfg.retainTags ?? []).map((t) => applyTemplate(t, resolvers, "retainTags").trim()).filter(Boolean).filter((t) => {
1199
+ if (!RESERVED_TAG_PREFIX.test(t)) return true;
1200
+ log.warn("retain-stamp", "ignoring reserved retainTags entry", { tag: t });
1201
+ return false;
1202
+ });
1203
+ const metadata = {};
1204
+ for (const [key, value] of Object.entries(cfg.retainMetadata ?? {})) {
1205
+ metadata[key] = applyTemplate(value, resolvers, "retainMetadata");
1206
+ }
1207
+ return { tags, metadata };
1208
+ }
1209
+
1210
+ // src/core/jsonl.ts
1211
+ import { closeSync, openSync, readSync, statSync } from "fs";
1212
+ import { StringDecoder } from "string_decoder";
1213
+ var CHUNK_BYTES = 64 * 1024;
1214
+ var MAX_TRANSCRIPT_BYTES = 32 * 1024 * 1024;
1215
+ function readJsonlTail(path, opts) {
1216
+ const maxBytes = opts.maxBytes ?? MAX_TRANSCRIPT_BYTES;
1217
+ let size;
1218
+ try {
1219
+ size = statSync(path).size;
1220
+ } catch {
1221
+ return { lines: emptyLines(), skippedBytes: 0 };
1222
+ }
1223
+ const skippedBytes = size > maxBytes ? size - maxBytes : 0;
1224
+ if (skippedBytes > 0) {
1225
+ log.warn(opts.scope, "transcript too large \u2014 retaining the most recent portion only", {
1226
+ path,
1227
+ sizeBytes: size,
1228
+ skippedBytes
1229
+ });
1230
+ }
1231
+ return { lines: streamLines(path, skippedBytes), skippedBytes };
1232
+ }
1233
+ function* emptyLines() {
1234
+ }
1235
+ function* streamLines(path, start) {
1236
+ let fd;
1237
+ try {
1238
+ fd = openSync(path, "r");
1239
+ } catch {
1240
+ return;
1241
+ }
1242
+ try {
1243
+ const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
1244
+ const decoder = new StringDecoder("utf8");
1245
+ let pending = "";
1246
+ let position = start > 0 ? start - 1 : 0;
1247
+ let dropPartial = start > 0;
1248
+ for (; ; ) {
1249
+ const bytesRead = readSync(fd, buffer, 0, buffer.length, position);
1250
+ if (bytesRead <= 0) break;
1251
+ position += bytesRead;
1252
+ pending += decoder.write(buffer.subarray(0, bytesRead));
1253
+ let newline;
1254
+ while ((newline = pending.indexOf("\n")) !== -1) {
1255
+ const line = pending.slice(0, newline);
1256
+ pending = pending.slice(newline + 1);
1257
+ if (dropPartial) {
1258
+ dropPartial = false;
1259
+ continue;
1260
+ }
1261
+ yield line;
1262
+ }
1263
+ }
1264
+ pending += decoder.end();
1265
+ if (pending && !dropPartial) yield pending;
1266
+ } finally {
1267
+ closeSync(fd);
1268
+ }
1269
+ }
949
1270
 
950
1271
  // src/core/transcript-util.ts
951
- var MEMORY_TAG_RE = /<(hook_prompt|hindsight_memory|hindsight_memories|hindsight_bank|relevant_memories|user_feedback|hindsight_knowledge|hindsight_knowledge_refresh)\b[\s\S]*?<\/\1>/g;
1272
+ var MEMORY_TAG_RE = /<(hook_prompt|task-notification|system-reminder|hindsight_memory|hindsight_memories|hindsight_bank|relevant_memories|user_feedback|hindsight_knowledge|hindsight_knowledge_refresh)\b[\s\S]*?<\/\1>/g;
952
1273
  function stripInjectedMemory(s) {
953
1274
  return s.replace(MEMORY_TAG_RE, "");
954
1275
  }
@@ -1007,14 +1328,8 @@ function renderLine(content, type) {
1007
1328
  return out;
1008
1329
  }
1009
1330
  function readClaudeTranscript(path) {
1010
- let raw;
1011
- try {
1012
- raw = readFileSync4(path, "utf8");
1013
- } catch {
1014
- return [];
1015
- }
1016
1331
  const turns = [];
1017
- for (const rawLine of raw.split("\n")) {
1332
+ for (const rawLine of readJsonlTail(path, { scope: "claude-code" }).lines) {
1018
1333
  const trimmed = rawLine.trim();
1019
1334
  if (!trimmed) continue;
1020
1335
  let parsed;
@@ -1047,7 +1362,10 @@ async function buildRetain(args) {
1047
1362
  const startTs = turns[0]?.timestamp ?? (/* @__PURE__ */ new Date()).toISOString();
1048
1363
  const t0 = Date.now();
1049
1364
  try {
1050
- await retainLiveSession(client, sessionId, turns, startTs, harness);
1365
+ await retainLiveSession(client, sessionId, turns, startTs, harness, {
1366
+ cursors: args.cursors ?? fileCursorStore(harness),
1367
+ stamp: args.stamp
1368
+ });
1051
1369
  diag(harness, "retain_ok", { ms: Date.now() - t0, turns: turns.length, session: sessionId });
1052
1370
  } catch (e) {
1053
1371
  log.warn(harness, "session write-back failed", {
@@ -1064,7 +1382,7 @@ async function runRetainHook(spec, makeClient = (o) => new HindsightClient(o)) {
1064
1382
  if (process.env.HINDSIGHT_DISABLE_HOOKS) return;
1065
1383
  let ev = {};
1066
1384
  try {
1067
- ev = JSON.parse(readFileSync5(0, "utf8"));
1385
+ ev = JSON.parse(readFileSync4(0, "utf8"));
1068
1386
  } catch {
1069
1387
  return;
1070
1388
  }
@@ -1085,12 +1403,18 @@ async function runRetainHook(spec, makeClient = (o) => new HindsightClient(o)) {
1085
1403
  sessionId: sessionId || "no-session",
1086
1404
  transcriptPath,
1087
1405
  client,
1088
- readTranscript: spec.readTranscript
1406
+ readTranscript: spec.readTranscript,
1407
+ stamp: buildRetainStamp(cfg, {
1408
+ directory: cwd,
1409
+ harness: spec.harness,
1410
+ bankId,
1411
+ sessionId: sessionId || "no-session"
1412
+ })
1089
1413
  });
1090
1414
  }
1091
1415
 
1092
1416
  // src/core/session-start.ts
1093
- import { readFileSync as readFileSync7 } from "fs";
1417
+ import { readFileSync as readFileSync6 } from "fs";
1094
1418
 
1095
1419
  // src/core/git.ts
1096
1420
  import { execFileSync as execFileSync3 } from "child_process";
@@ -1107,13 +1431,12 @@ import { delimiter, dirname as dirname6, join as join7 } from "path";
1107
1431
  import { fileURLToPath as fileURLToPath3 } from "url";
1108
1432
 
1109
1433
  // src/core/skill-sync.ts
1110
- import { cpSync, existsSync as existsSync3, readFileSync as readFileSync6 } from "fs";
1434
+ import { cpSync, existsSync as existsSync3, readFileSync as readFileSync5 } from "fs";
1111
1435
  import { homedir as homedir4 } from "os";
1112
1436
  import { dirname as dirname7, join as join8 } from "path";
1113
1437
  import { fileURLToPath as fileURLToPath4 } from "url";
1114
1438
 
1115
1439
  // src/core/transcript-codex.ts
1116
- import { readFileSync as readFileSync8 } from "fs";
1117
1440
  function isSyntheticUserText(text) {
1118
1441
  const s = text.trimStart();
1119
1442
  return s.startsWith("# AGENTS.md instructions for ") || s.startsWith("<environment_context>");
@@ -1122,14 +1445,8 @@ function messageText(payload) {
1122
1445
  return (payload.content || []).filter((c) => c && typeof c.text === "string").map((c) => c.text).join("\n");
1123
1446
  }
1124
1447
  function readCodexTranscript(path) {
1125
- let raw;
1126
- try {
1127
- raw = readFileSync8(path, "utf8");
1128
- } catch {
1129
- return [];
1130
- }
1131
1448
  const turns = [];
1132
- for (const rawLine of raw.split("\n")) {
1449
+ for (const rawLine of readJsonlTail(path, { scope: "codex" }).lines) {
1133
1450
  const trimmed = rawLine.trim();
1134
1451
  if (!trimmed) continue;
1135
1452
  let parsed;
@@ -1163,7 +1480,6 @@ function readCodexTranscript(path) {
1163
1480
  }
1164
1481
 
1165
1482
  // src/core/transcript-cursor.ts
1166
- import { readFileSync as readFileSync9 } from "fs";
1167
1483
  function withTimestamp(turn, timestamp) {
1168
1484
  return timestamp ? { ...turn, timestamp } : turn;
1169
1485
  }
@@ -1173,14 +1489,8 @@ function textFrom(content) {
1173
1489
  return content.filter((block) => block?.type === "text" && typeof block.text === "string").map((block) => block.text).join("\n");
1174
1490
  }
1175
1491
  function readCursorTranscript(path) {
1176
- let raw;
1177
- try {
1178
- raw = readFileSync9(path, "utf8");
1179
- } catch {
1180
- return [];
1181
- }
1182
1492
  const turns = [];
1183
- for (const rawLine of raw.split("\n")) {
1493
+ for (const rawLine of readJsonlTail(path, { scope: "cursor-cli" }).lines) {
1184
1494
  let parsed;
1185
1495
  try {
1186
1496
  parsed = JSON.parse(rawLine);
@@ -1216,17 +1526,10 @@ function readCursorTranscript(path) {
1216
1526
  }
1217
1527
 
1218
1528
  // src/core/transcript-antigravity.ts
1219
- import { readFileSync as readFileSync10 } from "fs";
1220
1529
  function readAntigravityTranscript(path) {
1221
1530
  if (!path) return [];
1222
- let raw;
1223
- try {
1224
- raw = readFileSync10(path, "utf8");
1225
- } catch {
1226
- return [];
1227
- }
1228
1531
  const turns = [];
1229
- for (const line of raw.split("\n")) {
1532
+ for (const line of readJsonlTail(path, { scope: "antigravity-cli" }).lines) {
1230
1533
  try {
1231
1534
  const event = JSON.parse(line);
1232
1535
  const role = event.role ?? event.message?.role ?? event.type;
@@ -1247,16 +1550,9 @@ function readAntigravityTranscript(path) {
1247
1550
  }
1248
1551
 
1249
1552
  // src/core/transcript-copilot.ts
1250
- import { readFileSync as readFileSync11 } from "fs";
1251
1553
  function readCopilotTranscript(path) {
1252
- let raw;
1253
- try {
1254
- raw = readFileSync11(path, "utf8");
1255
- } catch {
1256
- return [];
1257
- }
1258
1554
  const turns = [];
1259
- for (const rawLine of raw.split("\n")) {
1555
+ for (const rawLine of readJsonlTail(path, { scope: "copilot-cli" }).lines) {
1260
1556
  try {
1261
1557
  const event = JSON.parse(rawLine);
1262
1558
  const role = event.type === "user.message" ? "user" : event.type === "assistant.message" ? "assistant" : void 0;
@@ -1270,7 +1566,7 @@ function readCopilotTranscript(path) {
1270
1566
  }
1271
1567
 
1272
1568
  // src/core/transcript-grok.ts
1273
- import { existsSync as existsSync4, readFileSync as readFileSync12, readdirSync } from "fs";
1569
+ import { existsSync as existsSync4, readFileSync as readFileSync7, readdirSync } from "fs";
1274
1570
  import { homedir as homedir5 } from "os";
1275
1571
  import { join as join9 } from "path";
1276
1572
  var CHAT_HISTORY = "chat_history.jsonl";
@@ -1281,7 +1577,7 @@ function grokTranscriptPath(cwd, sessionId, grokHome = process.env.GROK_HOME ||
1281
1577
  try {
1282
1578
  for (const candidate of readdirSync(sessionsDir)) {
1283
1579
  const directory = join9(sessionsDir, candidate);
1284
- if (readFileSync12(join9(directory, ".cwd"), "utf8").trim() === cwd) {
1580
+ if (readFileSync7(join9(directory, ".cwd"), "utf8").trim() === cwd) {
1285
1581
  return join9(directory, sessionId, CHAT_HISTORY);
1286
1582
  }
1287
1583
  }
@@ -1290,14 +1586,8 @@ function grokTranscriptPath(cwd, sessionId, grokHome = process.env.GROK_HOME ||
1290
1586
  return direct;
1291
1587
  }
1292
1588
  function readGrokTranscript(path) {
1293
- let raw;
1294
- try {
1295
- raw = readFileSync12(path, "utf8");
1296
- } catch {
1297
- return [];
1298
- }
1299
1589
  const turns = [];
1300
- for (const rawLine of raw.split("\n")) {
1590
+ for (const rawLine of readJsonlTail(path, { scope: "grok-build" }).lines) {
1301
1591
  try {
1302
1592
  const event = JSON.parse(rawLine);
1303
1593
  if (event.type !== "user" && event.type !== "assistant") continue;