@vectorize-io/hindsight-coding-agents 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,11 @@
1
1
  #!/usr/bin/env node
2
2
 
3
3
  // src/core/hook.ts
4
- import { existsSync, readFileSync as readFileSync3 } from "fs";
4
+ import { existsSync as existsSync2, readFileSync as readFileSync3 } from "fs";
5
5
 
6
6
  // src/core/bank.ts
7
7
  import { execFileSync } from "child_process";
8
+ import { existsSync } from "fs";
8
9
  import { homedir } from "os";
9
10
  import { basename, dirname, join, normalize, sep } from "path";
10
11
 
@@ -40,12 +41,35 @@ function getProjectRootFromGit(directory) {
40
41
  return null;
41
42
  }
42
43
  }
44
+ function projectNameOf(directory) {
45
+ return gitProjectName(directory, true);
46
+ }
47
+ var PROJECT_ROOT_ENV = ["CLAUDE_PROJECT_DIR"];
48
+ function nearestExistingDir(directory) {
49
+ let current2 = directory;
50
+ while (current2) {
51
+ if (existsSync(current2)) return current2;
52
+ const parent = dirname(current2);
53
+ if (parent === current2) return "";
54
+ current2 = parent;
55
+ }
56
+ return "";
57
+ }
58
+ function dirName(directory) {
59
+ return directory && basename(directory) || "unknown";
60
+ }
43
61
  function gitProjectName(directory, resolveWorktrees) {
44
62
  if (resolveWorktrees) {
45
- const root = getProjectRootFromGit(directory);
46
- if (root) return basename(root);
63
+ const candidates = [
64
+ nearestExistingDir(directory),
65
+ ...PROJECT_ROOT_ENV.map((v) => process.env[v] || "")
66
+ ];
67
+ for (const candidate of candidates) {
68
+ const root = candidate ? getProjectRootFromGit(candidate) : null;
69
+ if (root) return basename(root);
70
+ }
47
71
  }
48
- return directory ? basename(directory) : "unknown";
72
+ return dirName(directory);
49
73
  }
50
74
  function mapLookup(map, directory) {
51
75
  const cwd = normalize(directory);
@@ -66,7 +90,7 @@ function deriveBankId(config, directory, harness = "coding") {
66
90
  if (!dynamic) return config.bankId || DEFAULT_BANK_NAME;
67
91
  const resolvers = {
68
92
  harness: () => harness,
69
- project: () => directory ? basename(directory) : "unknown",
93
+ project: () => dirName(directory),
70
94
  gitProject: () => gitProjectName(directory, config.resolveWorktrees ?? true),
71
95
  channel: () => process.env.HINDSIGHT_CHANNEL_ID || "default",
72
96
  user: () => process.env.HINDSIGHT_USER_ID || "anonymous"
@@ -140,7 +164,7 @@ function resolveConfig(raw = {}) {
140
164
  disabled: raw.disabled ?? false,
141
165
  retainSessions: raw.retainSessions ?? true,
142
166
  // opencode: write back by default (parity with hook-harness Stop)
143
- retainEveryTurns: raw.retainEveryTurns || 1,
167
+ maxParallelRetains: raw.maxParallelRetains || 10,
144
168
  reflectTimeoutMs: raw.reflectTimeoutMs || 12e4,
145
169
  autoReflect: raw.autoReflect ?? true,
146
170
  pageRefreshEveryTurns: raw.pageRefreshEveryTurns || 10,
@@ -199,7 +223,7 @@ var ENV_KEYS = {
199
223
  harness: "HINDSIGHT_HARNESS",
200
224
  disabled: "HINDSIGHT_DISABLED",
201
225
  retainSessions: "HINDSIGHT_RETAIN_SESSIONS",
202
- retainEveryTurns: "HINDSIGHT_RETAIN_EVERY_TURNS",
226
+ maxParallelRetains: "HINDSIGHT_MAX_PARALLEL_RETAINS",
203
227
  reflectTimeoutMs: "HINDSIGHT_REFLECT_TIMEOUT_MS",
204
228
  autoReflect: "HINDSIGHT_AUTO_REFLECT",
205
229
  pageRefreshEveryTurns: "HINDSIGHT_PAGE_REFRESH_EVERY_TURNS",
@@ -228,7 +252,7 @@ var ENV_LISTS = /* @__PURE__ */ new Set(["retainTags"]);
228
252
  var ENV_NUMBERS = /* @__PURE__ */ new Set([
229
253
  "apiPort",
230
254
  "daemonIdleTimeout",
231
- "retainEveryTurns",
255
+ "maxParallelRetains",
232
256
  "reflectTimeoutMs",
233
257
  "pageRefreshEveryTurns",
234
258
  "seedLimit",
@@ -482,9 +506,40 @@ function semverGte(version, min) {
482
506
  }
483
507
  return true;
484
508
  }
509
+ async function pool(items, n, fn, onError, onProgress) {
510
+ let i = 0, done = 0;
511
+ async function worker() {
512
+ while (i < items.length) {
513
+ const idx = i++;
514
+ try {
515
+ await fn(items[idx], idx);
516
+ } catch (e) {
517
+ onError?.(idx, e);
518
+ }
519
+ onProgress?.(++done, items.length);
520
+ }
521
+ }
522
+ await Promise.all(Array.from({ length: Math.min(n, items.length) }, worker));
523
+ }
485
524
 
486
525
  // src/core/hindsight.ts
487
526
  var MIN_IDEMPOTENT_RETAIN_VERSION = "0.8.6";
527
+ var RateLimitedError = class extends Error {
528
+ constructor(retryAfterMs2) {
529
+ super(`rate limited; retry after ${Math.round(retryAfterMs2 / 1e3)}s`);
530
+ this.retryAfterMs = retryAfterMs2;
531
+ this.name = "RateLimitedError";
532
+ }
533
+ retryAfterMs;
534
+ code = "rate_limited";
535
+ };
536
+ function retryAfterMs(header) {
537
+ if (!header) return 0;
538
+ const secs = Number(header.trim());
539
+ if (Number.isFinite(secs) && secs >= 0) return secs * 1e3;
540
+ const at = Date.parse(header);
541
+ return Number.isNaN(at) ? 0 : Math.max(0, at - Date.now());
542
+ }
488
543
  var KnowledgePagesUnavailableError = class extends Error {
489
544
  code = "knowledge_pages_unavailable";
490
545
  constructor() {
@@ -493,6 +548,10 @@ var KnowledgePagesUnavailableError = class extends Error {
493
548
  }
494
549
  };
495
550
  var TERMINAL = /* @__PURE__ */ new Set(["completed", "failed", "cancelled", "error"]);
551
+ var DEFAULT_MAX_PARALLEL_RETAINS = 10;
552
+ var POLL_CYCLE_MS = 5e3;
553
+ var RETRY_AFTER_FLOOR_MS = 10 * 1e3;
554
+ var RETRY_AFTER_CEILING_MS = 60 * 1e3;
496
555
  var MISSION_FIELDS = ["reflect_mission", "retain_mission", "observations_mission"];
497
556
  var HindsightClient = class {
498
557
  apiUrl;
@@ -505,12 +564,14 @@ var HindsightClient = class {
505
564
  /** Tri-state capability probe: unknown until the first append-mode retain, then cached. */
506
565
  idempotentRetain;
507
566
  log;
567
+ maxParallelRetains;
508
568
  constructor(o) {
509
569
  this.apiUrl = o.apiUrl.replace(/\/$/, "");
510
570
  this.apiToken = o.apiToken;
511
571
  this.bank = o.bank;
512
572
  this.log = o.log ?? (() => {
513
573
  });
574
+ this.maxParallelRetains = o.maxParallelRetains || DEFAULT_MAX_PARALLEL_RETAINS;
514
575
  }
515
576
  headers() {
516
577
  const h = { "Content-Type": "application/json" };
@@ -528,6 +589,8 @@ var HindsightClient = class {
528
589
  body: body ? JSON.stringify(body) : void 0,
529
590
  signal: AbortSignal.timeout(15e3)
530
591
  });
592
+ if (r.status === 429 && !tolerate.includes(429))
593
+ throw new RateLimitedError(retryAfterMs(r.headers.get("retry-after")));
531
594
  if (!r.ok && r.status !== 404 && !tolerate.includes(r.status))
532
595
  throw new Error(`${method} ${url} -> ${r.status} ${await r.text()}`);
533
596
  return r;
@@ -659,7 +722,14 @@ var HindsightClient = class {
659
722
  return 0;
660
723
  }
661
724
  }
662
- /** Poll each enqueued operation by id until terminal. LIST only shows active ops, so per-id GET is reliable. */
725
+ /**
726
+ * Poll each enqueued operation by id until terminal. LIST only shows active ops, so per-id GET is reliable.
727
+ *
728
+ * Concurrency is capped at `maxParallelRetains` (the API rate-limits bursts, not single
729
+ * requests — a 200 to a lone GET with 429s under `Promise.all` over every pending op). A 429
730
+ * leaves the op pending and backs the next cycle off by its `Retry-After` (10s floor) instead
731
+ * of hammering the next cycle 5s later.
732
+ */
663
733
  async drain(ids, label, maxMs = 60 * 60 * 1e3) {
664
734
  if (!ids.length) return;
665
735
  this.log(`[wait] draining ${ids.length} ${label} operations \u2026`);
@@ -667,23 +737,29 @@ var HindsightClient = class {
667
737
  const pending = new Set(ids);
668
738
  let failed = 0;
669
739
  while (pending.size && Date.now() - start < maxMs) {
670
- await Promise.all(
671
- [...pending].map(async (id) => {
672
- try {
673
- const r = await fetch(this.bankUrl(`/operations/${id}`), { headers: this.headers() });
674
- if (!r.ok) return;
675
- const st = ((await r.json()).status || "").toLowerCase();
676
- if (TERMINAL.has(st)) {
677
- pending.delete(id);
678
- if (st !== "completed") failed++;
679
- }
680
- } catch {
740
+ let backoffMs = POLL_CYCLE_MS;
741
+ await pool([...pending], this.maxParallelRetains, async (id) => {
742
+ try {
743
+ const r = await fetch(this.bankUrl(`/operations/${id}`), { headers: this.headers() });
744
+ if (r.status === 429) {
745
+ backoffMs = Math.min(
746
+ RETRY_AFTER_CEILING_MS,
747
+ Math.max(backoffMs, RETRY_AFTER_FLOOR_MS, retryAfterMs(r.headers.get("retry-after")))
748
+ );
749
+ return;
681
750
  }
682
- })
683
- );
751
+ if (!r.ok) return;
752
+ const st = ((await r.json()).status || "").toLowerCase();
753
+ if (TERMINAL.has(st)) {
754
+ pending.delete(id);
755
+ if (st !== "completed") failed++;
756
+ }
757
+ } catch {
758
+ }
759
+ });
684
760
  if (pending.size) {
685
761
  this.log(` \u2026 ${pending.size}/${ids.length} ${label} ops pending`);
686
- await sleep(5e3);
762
+ await sleep(backoffMs);
687
763
  }
688
764
  }
689
765
  this.log(
@@ -883,13 +959,20 @@ var HindsightClient = class {
883
959
  const verb = args.relatesToPageId ? "Enhancement to an existing initiative" : "New initiative";
884
960
  const content = `${verb}: ${args.title}. ${args.summary}`;
885
961
  const markerId = `initiative-marker-${this.slugify(args.title)}-${Date.now()}`;
886
- await this.retain(
887
- content,
888
- "initiative marker",
889
- markerId,
890
- ["knowledge:feature-work", `relatedPageId:${pageId}`],
891
- "document"
892
- );
962
+ const tags = [
963
+ .../* @__PURE__ */ new Set([
964
+ ...args.stamp?.tags ?? [],
965
+ "knowledge:feature-work",
966
+ `relatedPageId:${pageId}`
967
+ ])
968
+ ];
969
+ if (Object.keys(args.stamp?.metadata ?? {}).length) {
970
+ await this.retain(content, "initiative marker", markerId, tags, "document", {
971
+ metadata: args.stamp?.metadata
972
+ });
973
+ } else {
974
+ await this.retain(content, "initiative marker", markerId, tags, "document");
975
+ }
893
976
  return { page_id: pageId };
894
977
  }
895
978
  /** Find a root folder by name (case-insensitive) or create it; returns its id. Fail-open to undefined. */
@@ -962,16 +1045,29 @@ This tool guide and the page list are re-injected for you periodically as things
962
1045
  }
963
1046
 
964
1047
  // src/core/session-cache.ts
965
- import { mkdirSync as mkdirSync2, readFileSync as readFileSync2, writeFileSync } from "fs";
1048
+ import { mkdirSync as mkdirSync2, readFileSync as readFileSync2, renameSync, rmSync, writeFileSync } from "fs";
966
1049
  import { tmpdir as tmpdir2 } from "os";
967
1050
  import { dirname as dirname4, join as join5 } from "path";
968
1051
  function sessionCacheFile(harness, sessionId) {
969
1052
  return join5(tmpdir2(), `hindsight-${harness}`, `${sessionId}.json`);
970
1053
  }
1054
+ function writeFileAtomic(path, body) {
1055
+ mkdirSync2(dirname4(path), { recursive: true });
1056
+ const tmp = `${path}.${process.pid}.tmp`;
1057
+ try {
1058
+ writeFileSync(tmp, body);
1059
+ renameSync(tmp, path);
1060
+ } catch (e) {
1061
+ try {
1062
+ rmSync(tmp, { force: true });
1063
+ } catch {
1064
+ }
1065
+ throw e;
1066
+ }
1067
+ }
971
1068
  function writeSessionCache(cacheFile, cache) {
972
1069
  try {
973
- mkdirSync2(dirname4(cacheFile), { recursive: true });
974
- writeFileSync(cacheFile, JSON.stringify(cache));
1070
+ writeFileAtomic(cacheFile, JSON.stringify(cache));
975
1071
  } catch {
976
1072
  }
977
1073
  }
@@ -1103,6 +1199,50 @@ function preflightDaemon(cfg, harness) {
1103
1199
 
1104
1200
  // src/core/retain-stamp.ts
1105
1201
  import { basename as basename2 } from "path";
1202
+ var RESERVED_TAG_PREFIX = /^(source|harness):/;
1203
+ function resolversFor(ctx) {
1204
+ return {
1205
+ // Worktree-aware like the bank id, so every linked worktree of a repo stamps the SAME name —
1206
+ // otherwise a shared bank ends up with `project:app` and `project:app-wt2` for one repository.
1207
+ gitProject: () => projectNameOf(ctx.directory),
1208
+ project: () => ctx.directory ? basename2(ctx.directory) : "unknown",
1209
+ harness: () => ctx.harness,
1210
+ bankId: () => ctx.bankId,
1211
+ sessionId: () => ctx.sessionId ?? "unknown",
1212
+ timestamp: () => (/* @__PURE__ */ new Date()).toISOString(),
1213
+ channel: () => process.env.HINDSIGHT_CHANNEL_ID || "default",
1214
+ user: () => process.env.HINDSIGHT_USER_ID || "anonymous"
1215
+ };
1216
+ }
1217
+ function buildRetainStamp(cfg, ctx) {
1218
+ const hasTags = Boolean(cfg.retainTags?.length);
1219
+ const hasMetadata = Boolean(cfg.retainMetadata && Object.keys(cfg.retainMetadata).length);
1220
+ if (!hasTags && !hasMetadata) return { tags: [], metadata: {} };
1221
+ const base = resolversFor(ctx);
1222
+ const cache = /* @__PURE__ */ new Map();
1223
+ const resolvers = Object.fromEntries(
1224
+ Object.entries(base).map(([name, resolve2]) => [
1225
+ name,
1226
+ () => {
1227
+ const hit = cache.get(name);
1228
+ if (hit !== void 0) return hit;
1229
+ const value = resolve2();
1230
+ cache.set(name, value);
1231
+ return value;
1232
+ }
1233
+ ])
1234
+ );
1235
+ const tags = (cfg.retainTags ?? []).map((t) => applyTemplate(t, resolvers, "retainTags").trim()).filter(Boolean).filter((t) => {
1236
+ if (!RESERVED_TAG_PREFIX.test(t)) return true;
1237
+ log.warn("retain-stamp", "ignoring reserved retainTags entry", { tag: t });
1238
+ return false;
1239
+ });
1240
+ const metadata = {};
1241
+ for (const [key, value] of Object.entries(cfg.retainMetadata ?? {})) {
1242
+ metadata[key] = applyTemplate(value, resolvers, "retainMetadata");
1243
+ }
1244
+ return { tags, metadata };
1245
+ }
1106
1246
 
1107
1247
  // src/core/jsonl.ts
1108
1248
  import { closeSync, openSync, readSync, statSync } from "fs";
@@ -1237,7 +1377,7 @@ import { execFileSync as execFileSync4 } from "child_process";
1237
1377
 
1238
1378
  // src/core/survey.ts
1239
1379
  import { spawn as realSpawn3 } from "child_process";
1240
- import { accessSync, constants, existsSync as existsSync2 } from "fs";
1380
+ import { accessSync, constants, existsSync as existsSync3 } from "fs";
1241
1381
  import { homedir as homedir3 } from "os";
1242
1382
  import { delimiter, dirname as dirname6, join as join7 } from "path";
1243
1383
  import { fileURLToPath as fileURLToPath3 } from "url";
@@ -1252,7 +1392,7 @@ function resolveClaudeBin(explicit) {
1252
1392
  if (process.env.HINDSIGHT_CLAUDE_BIN) return process.env.HINDSIGHT_CLAUDE_BIN;
1253
1393
  const nativeInstallPath = join7(homedir3(), ".claude", "local", "claude");
1254
1394
  try {
1255
- if (existsSync2(nativeInstallPath)) return nativeInstallPath;
1395
+ if (existsSync3(nativeInstallPath)) return nativeInstallPath;
1256
1396
  } catch {
1257
1397
  }
1258
1398
  return "claude";
@@ -1426,7 +1566,7 @@ function startCodebaseSurvey(repoDir, opts = {}) {
1426
1566
  var DEEPEN_DIFF_TARGET = 300;
1427
1567
 
1428
1568
  // src/core/skill-sync.ts
1429
- import { cpSync, existsSync as existsSync3, readFileSync as readFileSync5 } from "fs";
1569
+ import { cpSync, existsSync as existsSync4, readFileSync as readFileSync5 } from "fs";
1430
1570
  import { homedir as homedir4 } from "os";
1431
1571
  import { dirname as dirname7, join as join8 } from "path";
1432
1572
  import { fileURLToPath as fileURLToPath4 } from "url";
@@ -1446,10 +1586,10 @@ function syncCompanionSkill(harness, opts = {}) {
1446
1586
  if (!parts) return;
1447
1587
  const src = opts.srcDir ?? packagedSkillDir();
1448
1588
  const srcMd = join8(src, "SKILL.md");
1449
- if (!existsSync3(srcMd)) return;
1589
+ if (!existsSync4(srcMd)) return;
1450
1590
  const dst = join8(opts.home ?? homedir4(), ...parts, "hindsight-coding-agent");
1451
1591
  const dstMd = join8(dst, "SKILL.md");
1452
- if (!existsSync3(dstMd)) return;
1592
+ if (!existsSync4(dstMd)) return;
1453
1593
  if (readFileSync5(srcMd, "utf8") !== readFileSync5(dstMd, "utf8")) {
1454
1594
  cpSync(src, dst, { recursive: true });
1455
1595
  }
@@ -1495,20 +1635,25 @@ async function buildSessionStartContext(args) {
1495
1635
  const startSurvey = args.startSurvey ?? startCodebaseSurvey;
1496
1636
  const resolveHeadSha = args.headSha ?? gitHeadSha;
1497
1637
  const countCommitsSince = args.commitsSince ?? commitsSince;
1638
+ const retainStamp = () => buildRetainStamp(cfg, { directory: cwd, harness, bankId });
1498
1639
  const SURVEY_BASELINE_TAG = "source:survey-baseline";
1499
1640
  const SURVEY_BASELINE_PREFIX = "survey-baseline:";
1500
1641
  const recordSurveyBaseline = (sha) => {
1501
1642
  if (!client.retain) return;
1502
1643
  try {
1503
- void Promise.resolve(
1504
- client.retain(
1505
- `\u{1F6F0}\uFE0F Hindsight is researching this codebase \u2014 survey started at commit ${sha.slice(0, 12)}. (Internal marker: no memories are extracted from this document.)`,
1506
- "hindsight codebase-survey baseline",
1507
- `${SURVEY_BASELINE_PREFIX}${sha}`,
1508
- [SURVEY_BASELINE_TAG],
1509
- "survey"
1510
- )
1511
- ).catch(() => {
1644
+ const stamp = retainStamp();
1645
+ const content = `\u{1F6F0}\uFE0F Hindsight is researching this codebase \u2014 survey started at commit ${sha.slice(0, 12)}. (Internal marker: no memories are extracted from this document.)`;
1646
+ const tags = [.../* @__PURE__ */ new Set([...stamp.tags, SURVEY_BASELINE_TAG])];
1647
+ const retained = client.retain(
1648
+ content,
1649
+ "hindsight codebase-survey baseline",
1650
+ `${SURVEY_BASELINE_PREFIX}${sha}`,
1651
+ tags,
1652
+ "survey",
1653
+ // `retain` only sets metadata when it is truthy, so an empty stamp sends none.
1654
+ { metadata: Object.keys(stamp.metadata).length ? stamp.metadata : void 0 }
1655
+ );
1656
+ void Promise.resolve(retained).catch(() => {
1512
1657
  });
1513
1658
  } catch {
1514
1659
  }
@@ -1619,7 +1764,12 @@ async function runSessionStartHook(spec, makeClient = (o) => new HindsightClient
1619
1764
  const bankId = resolved.bankId;
1620
1765
  if (cfg.disabled) return;
1621
1766
  await ensureDaemon(cfg, harness, { waitMs: DAEMON_WAIT_SESSION_START_MS });
1622
- const client = makeClient({ apiUrl: cfg.apiUrl, apiToken: cfg.apiToken, bank: bankId });
1767
+ const client = makeClient({
1768
+ apiUrl: cfg.apiUrl,
1769
+ apiToken: cfg.apiToken,
1770
+ bank: bankId,
1771
+ maxParallelRetains: cfg.maxParallelRetains
1772
+ });
1623
1773
  const out = await buildSessionStartContext({ cwd, bankId, cfg, client, harness });
1624
1774
  if (out.deferInitialReflect && sessionId) {
1625
1775
  writeSessionCache(sessionCacheFile(harness, sessionId), { deferInitialReflect: true });
@@ -1762,14 +1912,14 @@ function readCopilotTranscript(path) {
1762
1912
  }
1763
1913
 
1764
1914
  // src/core/transcript-grok.ts
1765
- import { existsSync as existsSync4, readFileSync as readFileSync7, readdirSync } from "fs";
1915
+ import { existsSync as existsSync5, readFileSync as readFileSync7, readdirSync } from "fs";
1766
1916
  import { homedir as homedir5 } from "os";
1767
1917
  import { join as join9 } from "path";
1768
1918
  var CHAT_HISTORY = "chat_history.jsonl";
1769
1919
  function grokTranscriptPath(cwd, sessionId, grokHome = process.env.GROK_HOME || join9(homedir5(), ".grok")) {
1770
1920
  const sessionsDir = join9(grokHome, "sessions");
1771
1921
  const direct = join9(sessionsDir, encodeURIComponent(cwd), sessionId, CHAT_HISTORY);
1772
- if (existsSync4(direct)) return direct;
1922
+ if (existsSync5(direct)) return direct;
1773
1923
  try {
1774
1924
  for (const candidate of readdirSync(sessionsDir)) {
1775
1925
  const directory = join9(sessionsDir, candidate);
@@ -1809,7 +1959,7 @@ function readGrokTranscript(path) {
1809
1959
  }
1810
1960
 
1811
1961
  // src/core/transcript-devin.ts
1812
- import { existsSync as existsSync5 } from "fs";
1962
+ import { existsSync as existsSync6 } from "fs";
1813
1963
  import { createRequire } from "module";
1814
1964
  import { homedir as homedir6 } from "os";
1815
1965
  import { join as join10 } from "path";
@@ -1851,7 +2001,7 @@ function readDevinTranscript(sessionId, dbPath = devinSessionDb()) {
1851
2001
  diag("devin-cli", "sqlite_unavailable", { node: process.version, dbPath });
1852
2002
  return [];
1853
2003
  }
1854
- if (!existsSync5(dbPath)) {
2004
+ if (!existsSync6(dbPath)) {
1855
2005
  diag("devin-cli", "session_db_missing", { dbPath });
1856
2006
  return [];
1857
2007
  }