@midhunxavier/code-flow 0.0.0-stage → 0.1.0-preview.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (319) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +141 -2
  3. package/bin/codeflow.mjs +8 -0
  4. package/bin/launcher.mjs +127 -0
  5. package/dist/remote-host.json +1 -0
  6. package/licenses/inter.txt +92 -0
  7. package/licenses/preact.txt +21 -0
  8. package/licenses/signals-core.txt +21 -0
  9. package/licenses/signals.txt +21 -0
  10. package/licenses/typescript-notices.txt +193 -0
  11. package/licenses/typescript.txt +55 -0
  12. package/licenses/uqr.txt +22 -0
  13. package/licenses/xterm-fit.txt +19 -0
  14. package/licenses/xterm.txt +21 -0
  15. package/package.json +34 -4
  16. package/public/assets/changes-62ZIEQSN.js +1 -0
  17. package/public/assets/chunk-2PZVOKID.js +1 -0
  18. package/public/assets/chunk-4D5IENEB.js +1 -0
  19. package/public/assets/chunk-5SARP5MU.js +1 -0
  20. package/public/assets/chunk-63YLZ3PR.js +1 -0
  21. package/public/assets/chunk-7FUHKF2M.js +1 -0
  22. package/public/assets/chunk-APQEVJ2K.js +1 -0
  23. package/public/assets/chunk-CGOZQMEZ.js +2 -0
  24. package/public/assets/chunk-J4VYS3KV.js +2 -0
  25. package/public/assets/chunk-L7TUCIYF.js +1 -0
  26. package/public/assets/chunk-SIHDWM2Y.js +154 -0
  27. package/public/assets/chunk-TEJVYIKC.js +1 -0
  28. package/public/assets/code-browser-VB2HGZSG.js +4 -0
  29. package/public/assets/filtered-tasks-ES5CP4KV.js +1 -0
  30. package/public/assets/interactive-browser-7F5ZXRUV.js +7 -0
  31. package/public/assets/join-project-6RTFHNCA.js +1 -0
  32. package/public/assets/main-65EW73SB.js +1 -0
  33. package/public/assets/project-invitations-view-GMPWGWYM.js +1 -0
  34. package/public/assets/settings-UB5Q4AQM.js +9 -0
  35. package/public/assets/styles-18792BB4.css +32 -0
  36. package/public/assets/team-server-wizard-2GLDWS6D.js +8 -0
  37. package/public/assets/terminal-IWAIM4DE.js +62 -0
  38. package/public/assets.json +1 -0
  39. package/public/bundle-meta.json +1 -0
  40. package/public/docs/.nojekyll +0 -0
  41. package/public/docs/accessibility.html +19 -0
  42. package/public/docs/acp-agents.html +117 -0
  43. package/public/docs/agent-containers.html +79 -0
  44. package/public/docs/api.html +1130 -0
  45. package/public/docs/app.js +342 -0
  46. package/public/docs/application-improvements.html +275 -0
  47. package/public/docs/architecture-explorer.html +19 -0
  48. package/public/docs/architecture.html +149 -0
  49. package/public/docs/auto-team.html +148 -0
  50. package/public/docs/automations.html +109 -0
  51. package/public/docs/checkpoint-review.html +37 -0
  52. package/public/docs/code-hosts.html +35 -0
  53. package/public/docs/code-intelligence.html +35 -0
  54. package/public/docs/configuration.html +346 -0
  55. package/public/docs/context-inspection.html +23 -0
  56. package/public/docs/coordinated-changes.html +28 -0
  57. package/public/docs/deferred-runs.html +25 -0
  58. package/public/docs/dependency-discovery.html +19 -0
  59. package/public/docs/documentation.html +84 -0
  60. package/public/docs/dynamic-teams-evaluation.html +174 -0
  61. package/public/docs/dynamic-teams-implementation-plan.html +882 -0
  62. package/public/docs/dynamic-teams-implementation-review-1.html +141 -0
  63. package/public/docs/dynamic-teams-plan-review-0.html +204 -0
  64. package/public/docs/extensions.html +42 -0
  65. package/public/docs/external-sources.html +96 -0
  66. package/public/docs/filesystem-policy.html +26 -0
  67. package/public/docs/filtered-tasks.html +44 -0
  68. package/public/docs/font-license.txt +92 -0
  69. package/public/docs/getting-started.html +143 -0
  70. package/public/docs/github-review-threads.html +26 -0
  71. package/public/docs/github.html +157 -0
  72. package/public/docs/help.html +69 -0
  73. package/public/docs/index.html +88 -0
  74. package/public/docs/installation.html +80 -0
  75. package/public/docs/launch-profiles.html +48 -0
  76. package/public/docs/maintenance.html +60 -0
  77. package/public/docs/memory-claims.html +31 -0
  78. package/public/docs/memory-storage.html +58 -0
  79. package/public/docs/notifications.html +39 -0
  80. package/public/docs/patterns.html +195 -0
  81. package/public/docs/performance-results.html +210 -0
  82. package/public/docs/performance.html +166 -0
  83. package/public/docs/project-conventions.html +22 -0
  84. package/public/docs/project-first-navigation-plan.html +587 -0
  85. package/public/docs/project-teams-implementation-plan.html +772 -0
  86. package/public/docs/project-teams.html +129 -0
  87. package/public/docs/provider-compatibility.html +20 -0
  88. package/public/docs/releasing.html +65 -0
  89. package/public/docs/remote-workers.html +64 -0
  90. package/public/docs/review-inbox.html +26 -0
  91. package/public/docs/run-storage.html +23 -0
  92. package/public/docs/runs.html +141 -0
  93. package/public/docs/search.json +1 -0
  94. package/public/docs/security.html +133 -0
  95. package/public/docs/shared-projects.html +31 -0
  96. package/public/docs/ssh-projects.html +60 -0
  97. package/public/docs/style.css +768 -0
  98. package/public/docs/task-reruns.html +30 -0
  99. package/public/docs/team-collaboration.html +412 -0
  100. package/public/docs/verification.html +22 -0
  101. package/public/docs/vscode.html +53 -0
  102. package/public/docs/worker-permission-verification.html +131 -0
  103. package/public/index.html +17 -0
  104. package/scripts/check-node.mjs +7 -0
  105. package/scripts/codeflow.mjs +55 -0
  106. package/scripts/computer-use.swift +64 -0
  107. package/scripts/pty.py +273 -0
  108. package/src/acp-connections.js +110 -0
  109. package/src/adapters/acp-presets.js +37 -0
  110. package/src/adapters/acp-types.js +0 -0
  111. package/src/adapters/acp.js +328 -0
  112. package/src/adapters/agent-tools.js +28 -0
  113. package/src/adapters/agy-auth.js +67 -0
  114. package/src/adapters/agy-isolation.js +242 -0
  115. package/src/adapters/agy-tools.js +45 -0
  116. package/src/adapters/agy.js +326 -0
  117. package/src/adapters/claude-stream.js +58 -0
  118. package/src/adapters/claude.js +282 -0
  119. package/src/adapters/codex-app-server.js +555 -0
  120. package/src/adapters/codex.js +311 -0
  121. package/src/adapters/errors.js +57 -0
  122. package/src/adapters/index.js +91 -0
  123. package/src/adapters/managed-permissions.js +37 -0
  124. package/src/adapters/model-discovery.js +100 -0
  125. package/src/adapters/process.js +433 -0
  126. package/src/adapters/team-tools.js +103 -0
  127. package/src/adapters/types.js +0 -0
  128. package/src/agent-browser.js +688 -0
  129. package/src/agent-computer.js +88 -0
  130. package/src/agent-container.js +155 -0
  131. package/src/agent-team-definitions.js +131 -0
  132. package/src/agent-tool-client.mjs +79 -0
  133. package/src/agent-tools.js +296 -0
  134. package/src/approvals.js +134 -0
  135. package/src/architecture.js +456 -0
  136. package/src/assets.js +44 -0
  137. package/src/automation.js +901 -0
  138. package/src/browser-files.js +293 -0
  139. package/src/browser-ownership.js +150 -0
  140. package/src/certificate-renewal.js +75 -0
  141. package/src/chat-family.js +25 -0
  142. package/src/chat-handoff.js +87 -0
  143. package/src/checkpoint-review.js +72 -0
  144. package/src/checks.js +303 -0
  145. package/src/code-host-git.js +102 -0
  146. package/src/code-hosts-api.js +99 -0
  147. package/src/code-hosts.js +482 -0
  148. package/src/code-intelligence-worker.mjs +452 -0
  149. package/src/code-intelligence.js +141 -0
  150. package/src/collab/api.js +365 -0
  151. package/src/collab/checkpoints.js +198 -0
  152. package/src/collab/connections.js +139 -0
  153. package/src/collab/host-execution.js +347 -0
  154. package/src/collab/hub-client.js +210 -0
  155. package/src/collab/link-api.js +229 -0
  156. package/src/collab/link.js +595 -0
  157. package/src/collab/model.js +496 -0
  158. package/src/collab/runner.js +1014 -0
  159. package/src/collab/store.js +1452 -0
  160. package/src/collab/terminals.js +361 -0
  161. package/src/collaboration-reviews.js +116 -0
  162. package/src/config.js +48 -0
  163. package/src/context-budget.js +14 -0
  164. package/src/context-inspection.js +204 -0
  165. package/src/context.js +154 -0
  166. package/src/conventions.js +283 -0
  167. package/src/coordinated-api.js +247 -0
  168. package/src/deferred-launch.js +82 -0
  169. package/src/deferred-runs.js +515 -0
  170. package/src/dependency-health.js +35 -0
  171. package/src/diff.js +43 -0
  172. package/src/docs-help.js +316 -0
  173. package/src/doctor.js +232 -0
  174. package/src/documentation-changes.js +20 -0
  175. package/src/engine.js +5258 -0
  176. package/src/env.js +44 -0
  177. package/src/execution-budget.js +8 -0
  178. package/src/experience.js +50 -0
  179. package/src/extensions.js +144 -0
  180. package/src/filesystem-access.js +107 -0
  181. package/src/filesystem-policy.js +238 -0
  182. package/src/filtered-task-limits.js +93 -0
  183. package/src/filtered-tasks-api.js +53 -0
  184. package/src/filtered-tasks.js +576 -0
  185. package/src/frame-policy.js +66 -0
  186. package/src/fs.js +57 -0
  187. package/src/git.js +626 -0
  188. package/src/github-review-threads.js +181 -0
  189. package/src/github.js +538 -0
  190. package/src/http.js +69 -0
  191. package/src/improve-settings.js +38 -0
  192. package/src/improve.js +187 -0
  193. package/src/integrations.js +373 -0
  194. package/src/integrity.js +259 -0
  195. package/src/journey-api.js +77 -0
  196. package/src/journey-replay.js +65 -0
  197. package/src/journey-store.js +97 -0
  198. package/src/journeys.js +60 -0
  199. package/src/json.js +81 -0
  200. package/src/launch-profiles.js +292 -0
  201. package/src/learnings.js +80 -0
  202. package/src/local-app-url.js +19 -0
  203. package/src/log.js +39 -0
  204. package/src/maintenance.js +335 -0
  205. package/src/memory-analysis.js +182 -0
  206. package/src/memory-claims.js +153 -0
  207. package/src/memory-sqlite.js +236 -0
  208. package/src/memory.js +552 -0
  209. package/src/merge-validation.js +58 -0
  210. package/src/model-catalog.js +239 -0
  211. package/src/model-research.js +91 -0
  212. package/src/notes.js +176 -0
  213. package/src/notifications.js +413 -0
  214. package/src/patterns/auto-team-model.js +230 -0
  215. package/src/patterns/auto-team.js +816 -0
  216. package/src/patterns/council.js +199 -0
  217. package/src/patterns/debate.js +188 -0
  218. package/src/patterns/findings.js +141 -0
  219. package/src/patterns/index.js +21 -0
  220. package/src/patterns/loop.js +158 -0
  221. package/src/patterns/pipeline.js +125 -0
  222. package/src/patterns/prompts.js +181 -0
  223. package/src/patterns/race.js +147 -0
  224. package/src/patterns/schemas.js +40 -0
  225. package/src/patterns/solo.js +46 -0
  226. package/src/patterns/supervisor.js +235 -0
  227. package/src/patterns/task-draft.js +29 -0
  228. package/src/patterns/task-review.js +45 -0
  229. package/src/patterns/tdd.js +231 -0
  230. package/src/patterns/team-strategies.js +67 -0
  231. package/src/performance.js +43 -0
  232. package/src/persistence-status.js +35 -0
  233. package/src/phone-sessions.js +96 -0
  234. package/src/preview-overlay.js +392 -0
  235. package/src/preview.js +230 -0
  236. package/src/privacy.js +23 -0
  237. package/src/project-report.js +22 -0
  238. package/src/project-team-budget.js +22 -0
  239. package/src/project-team-model.js +225 -0
  240. package/src/project-team-policy.js +42 -0
  241. package/src/project-team-store.js +234 -0
  242. package/src/project-team-usage.js +24 -0
  243. package/src/project-teams-api.js +43 -0
  244. package/src/project-teams.js +644 -0
  245. package/src/projects.js +290 -0
  246. package/src/provider-eligibility.js +9 -0
  247. package/src/readiness.js +98 -0
  248. package/src/remote-tls.js +66 -0
  249. package/src/remote-workers/bundles.js +198 -0
  250. package/src/remote-workers/client.js +332 -0
  251. package/src/remote-workers/durability.js +37 -0
  252. package/src/remote-workers/protocol.js +68 -0
  253. package/src/remote-workers/stdio.js +138 -0
  254. package/src/remote-workers/worker.js +304 -0
  255. package/src/remote.js +204 -0
  256. package/src/repo-config.js +201 -0
  257. package/src/repo-coordination.js +725 -0
  258. package/src/repo-tree.js +106 -0
  259. package/src/repository-identity.js +13 -0
  260. package/src/retention-policy.js +466 -0
  261. package/src/review-inbox.js +38 -0
  262. package/src/routing.js +49 -0
  263. package/src/run-analysis.js +111 -0
  264. package/src/run-cold-store.js +231 -0
  265. package/src/run-environment.js +15 -0
  266. package/src/run-history.js +91 -0
  267. package/src/run-log.js +160 -0
  268. package/src/run-permissions.js +111 -0
  269. package/src/run-projection.js +77 -0
  270. package/src/run-recovery.js +53 -0
  271. package/src/run-status.js +56 -0
  272. package/src/serve.js +138 -0
  273. package/src/server.js +2871 -0
  274. package/src/shared-access.js +103 -0
  275. package/src/shared-accounts.js +381 -0
  276. package/src/shared-api.js +239 -0
  277. package/src/shell.js +346 -0
  278. package/src/ship.js +344 -0
  279. package/src/slots.js +74 -0
  280. package/src/snapshot.js +388 -0
  281. package/src/source-context.js +17 -0
  282. package/src/sse-writer.js +70 -0
  283. package/src/ssh-projects/bootstrap.mjs +95 -0
  284. package/src/ssh-projects/config.js +45 -0
  285. package/src/ssh-projects/gateway.js +143 -0
  286. package/src/ssh-projects/manager.js +320 -0
  287. package/src/ssh-projects/team-autostart.mjs +93 -0
  288. package/src/ssh-projects/team-remove.mjs +69 -0
  289. package/src/ssh-projects/team-status.mjs +39 -0
  290. package/src/ssh-projects/transport.js +133 -0
  291. package/src/stats.js +139 -0
  292. package/src/store.js +544 -0
  293. package/src/stream.js +91 -0
  294. package/src/sweeps.js +241 -0
  295. package/src/tailscale.js +101 -0
  296. package/src/task-draft.js +23 -0
  297. package/src/task-team.js +164 -0
  298. package/src/team-evaluation.js +243 -0
  299. package/src/team-mailbox.js +160 -0
  300. package/src/team-memory.js +84 -0
  301. package/src/team-rerun-graph.js +15 -0
  302. package/src/team-reruns.js +130 -0
  303. package/src/team-scheduler.js +121 -0
  304. package/src/team-steering.js +21 -0
  305. package/src/team-template-store.js +98 -0
  306. package/src/team-templates.js +117 -0
  307. package/src/team-tool-client.mjs +79 -0
  308. package/src/team-tools.js +235 -0
  309. package/src/teams.js +0 -0
  310. package/src/templates.js +62 -0
  311. package/src/transcript.js +134 -0
  312. package/src/verification-integrity.js +27 -0
  313. package/src/verification.js +183 -0
  314. package/src/watch.js +195 -0
  315. package/src/work-queue.js +47 -0
  316. package/src/workflow-control.js +83 -0
  317. package/src/workspace-api.js +207 -0
  318. package/src/workspace-changes.js +61 -0
  319. package/src/workspace.js +146 -0
package/src/store.js ADDED
@@ -0,0 +1,544 @@
1
+ import { execFile } from "node:child_process";
2
+ import { createHash, randomBytes } from "node:crypto";
3
+ import { createReadStream, existsSync, readFileSync, unlinkSync } from "node:fs";
4
+ import { appendFile, link, mkdir, open, readdir, readFile, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
5
+ import { uptime } from "node:os";
6
+ import { createInterface } from "node:readline";
7
+ import { join } from "node:path";
8
+ import { CODEFLOW_HOME } from "./config.js";
9
+ import { childEnv } from "./env.js";
10
+ import { logError, logWarn } from "./log.js";
11
+ import { WorkQueue } from "./work-queue.js";
12
+ const RUNS_DIR = join(CODEFLOW_HOME, "runs");
13
+ const JOURNAL_VERSION = 2;
14
+ const ready = mkdir(RUNS_DIR, { recursive: true });
15
+ const writes = new WorkQueue(4), checkpoints = new WorkQueue(1);
16
+ const runKey = (file) => file.replace(/(?:\.journal)?\.jsonl$/, "");
17
+ let queuedBytes = 0;
18
+ const WRITE_BUFFER_BYTES = 64 * 1024 * 1024;
19
+ const writeQueueStats = () => ({ writes: { ...writes.stats, bytes: queuedBytes }, checkpoints: checkpoints.stats });
20
+ const appendBatches = /* @__PURE__ */ new Map();
21
+ const writeHealth = { failures: 0 };
22
+ const checkpointHealth = { failures: 0 };
23
+ const MAX_CHECKPOINT_BYTES = 16 * 1024 * 1024;
24
+ const checkpointId = (file) => /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,199}\.jsonl$/.test(file) && !file.endsWith(".journal.jsonl") ? file.slice(0, -6) : void 0;
25
+ const sourceIdentity = (info) => ({ size: String(info.size), mtimeNs: String(info.mtimeNs), ctimeNs: String(info.ctimeNs), dev: String(info.dev), ino: String(info.ino) });
26
+ const checksum = (text) => createHash("sha256").update(text).digest("hex");
27
+ async function writeRunCheckpoint(file, seq, runJson, source) {
28
+ const id = checkpointId(file);
29
+ if (!id) return;
30
+ const payload = `${JSON.stringify({ version: 1, source, seq }).slice(0, -1)},"run":${runJson}}`;
31
+ const text = `${payload.slice(0, -1)},"checksum":"${checksum(payload)}"}
32
+ `;
33
+ if (Buffer.byteLength(text) > MAX_CHECKPOINT_BYTES) return;
34
+ const path = join(RUNS_DIR, `${id}.checkpoint.json`);
35
+ const temp = `${path}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
36
+ try {
37
+ await writeFile(temp, text, { mode: 384, flag: "wx" });
38
+ if (JSON.stringify(sourceIdentity(await stat(join(RUNS_DIR, file), { bigint: true }))) !== JSON.stringify(source)) return;
39
+ await rename(temp, path);
40
+ } finally {
41
+ await unlink(temp).catch(() => void 0);
42
+ }
43
+ }
44
+ async function readRunCheckpoint(id) {
45
+ if (checkpointId(`${id}.jsonl`) !== id) return void 0;
46
+ const sourcePath = join(RUNS_DIR, `${id}.jsonl`);
47
+ try {
48
+ const before = sourceIdentity(await stat(sourcePath, { bigint: true }));
49
+ const handle = await open(join(RUNS_DIR, `${id}.checkpoint.json`), "r");
50
+ let text;
51
+ try {
52
+ const size = (await handle.stat()).size;
53
+ if (size === 0 || size > MAX_CHECKPOINT_BYTES) return void 0;
54
+ text = await handle.readFile("utf8");
55
+ } finally {
56
+ await handle.close();
57
+ }
58
+ if (Buffer.byteLength(text) > MAX_CHECKPOINT_BYTES) return void 0;
59
+ const cached = JSON.parse(text);
60
+ if (cached?.version !== 1 || !Number.isSafeInteger(cached.seq) || cached.seq < 0 || !cached.source || typeof cached.checksum !== "string" || !cached.run || typeof cached.run !== "object" || Array.isArray(cached.run) || cached.run.id !== id || !Array.isArray(cached.run.steps)) return void 0;
61
+ const payload = JSON.stringify({ version: cached.version, source: cached.source, seq: cached.seq, run: cached.run });
62
+ if (checksum(payload) !== cached.checksum) return void 0;
63
+ const after = sourceIdentity(await stat(sourcePath, { bigint: true }));
64
+ if (JSON.stringify(before) !== JSON.stringify(after) || JSON.stringify(cached.source) !== JSON.stringify(after)) return void 0;
65
+ return { run: cached.run, seq: cached.seq };
66
+ } catch {
67
+ return void 0;
68
+ }
69
+ }
70
+ const repaired = /* @__PURE__ */ new Set();
71
+ const unsynced = /* @__PURE__ */ new Set();
72
+ const directorySynced = /* @__PURE__ */ new Set();
73
+ async function endsWithNewline(path) {
74
+ const handle = await open(path, "r").catch(() => void 0);
75
+ if (!handle) return true;
76
+ try {
77
+ const { size } = await handle.stat();
78
+ if (!size) return true;
79
+ const buf = Buffer.alloc(1);
80
+ await handle.read(buf, 0, 1, size - 1);
81
+ return buf[0] === 10;
82
+ } finally {
83
+ await handle.close();
84
+ }
85
+ }
86
+ async function appendSynced(path, text) {
87
+ const handle = await open(path, "a");
88
+ try {
89
+ await handle.writeFile(text);
90
+ await handle.sync();
91
+ } finally {
92
+ await handle.close();
93
+ }
94
+ if (!directorySynced.has(path)) {
95
+ if (process.platform !== "win32") {
96
+ const directory = await open(RUNS_DIR, "r");
97
+ try {
98
+ await directory.sync();
99
+ } finally {
100
+ await directory.close();
101
+ }
102
+ }
103
+ directorySynced.add(path);
104
+ }
105
+ }
106
+ async function truncateIncompleteTail(path) {
107
+ const handle = await open(path, "r+").catch((err) => {
108
+ if (err.code === "ENOENT") return void 0;
109
+ throw err;
110
+ });
111
+ if (!handle) return;
112
+ try {
113
+ const { size } = await handle.stat();
114
+ if (!size) return;
115
+ let end = size;
116
+ const buffer = Buffer.alloc(Math.min(size, 64 * 1024));
117
+ while (end > 0) {
118
+ const start = Math.max(0, end - buffer.length);
119
+ const length = end - start;
120
+ await handle.read(buffer, 0, length, start);
121
+ const newline = buffer.subarray(0, length).lastIndexOf(10);
122
+ if (newline >= 0) {
123
+ const boundary = start + newline + 1;
124
+ if (boundary !== size) {
125
+ await handle.truncate(boundary);
126
+ await handle.sync();
127
+ }
128
+ return;
129
+ }
130
+ end = start;
131
+ }
132
+ await handle.truncate(0);
133
+ await handle.sync();
134
+ } finally {
135
+ await handle.close();
136
+ }
137
+ }
138
+ function appendLine(file, value, opts = {}) {
139
+ const line = `${JSON.stringify(value)}
140
+ `;
141
+ let checkpointWork;
142
+ let checkpoint;
143
+ if (opts.checkpoint && Number.isSafeInteger(opts.checkpoint.seq) && opts.checkpoint.seq >= 0 && typeof value === "object" && value && "seq" in value && value.seq === opts.checkpoint.seq) {
144
+ try {
145
+ const runJson = JSON.stringify(opts.checkpoint.run);
146
+ if (runJson && Buffer.byteLength(runJson) <= MAX_CHECKPOINT_BYTES) checkpoint = { seq: opts.checkpoint.seq, runJson };
147
+ } catch {
148
+ }
149
+ }
150
+ const key = runKey(file), size = Buffer.byteLength(line);
151
+ const previous = appendBatches.get(key);
152
+ const batchable = !opts.sync && !opts.strict && !opts.checkpoint;
153
+ let written;
154
+ if (queuedBytes + size > WRITE_BUFFER_BYTES) {
155
+ appendBatches.delete(key);
156
+ written = Promise.reject(new Error("Run write buffer is full; storage cannot keep up"));
157
+ } else if (batchable && previous?.file === file && previous.bytes + size <= 256 * 1024 && previous.count < 128) {
158
+ previous.text += line;
159
+ previous.bytes += size;
160
+ previous.count++;
161
+ queuedBytes += size;
162
+ written = previous.promise;
163
+ } else {
164
+ appendBatches.delete(key);
165
+ const batch = { file, text: line, bytes: size, count: 1, promise: Promise.resolve() };
166
+ queuedBytes += size;
167
+ written = writes.run(key, async () => {
168
+ if (appendBatches.get(key) === batch) appendBatches.delete(key);
169
+ await ready;
170
+ const path = join(RUNS_DIR, file);
171
+ if (opts.strict && !repaired.has(file)) await truncateIncompleteTail(path);
172
+ const prefix = repaired.has(file) || await endsWithNewline(path) ? "" : "\n";
173
+ try {
174
+ if (opts.sync) await appendSynced(path, prefix + batch.text);
175
+ else await appendFile(path, prefix + batch.text);
176
+ } catch (err) {
177
+ repaired.delete(file);
178
+ throw err;
179
+ }
180
+ repaired.add(file);
181
+ if (opts.sync) unsynced.delete(file);
182
+ else unsynced.add(file);
183
+ if (checkpoint && checkpoints.stats.bytes + Buffer.byteLength(checkpoint.runJson) <= 32 * 1024 * 1024) {
184
+ const captured = checkpoint;
185
+ const source = await stat(path, { bigint: true }).then(sourceIdentity).catch(() => void 0);
186
+ if (source) checkpointWork = checkpoints.run(key, () => writeRunCheckpoint(file, captured.seq, captured.runJson, source), Buffer.byteLength(captured.runJson)).catch((error) => {
187
+ checkpointHealth.failures++;
188
+ checkpointHealth.lastError = error.message;
189
+ checkpointHealth.at = Date.now();
190
+ if (checkpointHealth.failures === 1 || checkpointHealth.failures % 100 === 0) logWarn("couldn't save optional run checkpoint", { file, error: error.message });
191
+ });
192
+ }
193
+ }, size).finally(() => {
194
+ queuedBytes -= batch.bytes;
195
+ });
196
+ batch.promise = written;
197
+ if (batchable) appendBatches.set(key, batch);
198
+ }
199
+ const complete = written.then(() => checkpointWork);
200
+ const observed = complete.catch((err) => {
201
+ opts.onFailed?.();
202
+ writeHealth.failures++;
203
+ writeHealth.lastError = err.message;
204
+ writeHealth.at = Date.now();
205
+ if (writeHealth.failures === 1 || writeHealth.failures % 100 === 0) {
206
+ logError(`couldn't save run data (${writeHealth.failures} failed writes)`, err, { file });
207
+ }
208
+ });
209
+ return opts.strict ? complete : observed;
210
+ }
211
+ async function syncFile(file) {
212
+ const handle = await open(join(RUNS_DIR, file), "r+");
213
+ try {
214
+ await handle.sync();
215
+ } finally {
216
+ await handle.close();
217
+ }
218
+ }
219
+ async function flushWrites(id) {
220
+ if (id === void 0) appendBatches.clear();
221
+ else appendBatches.delete(id);
222
+ await writes.flush(id);
223
+ const files = [...unsynced].filter((file) => id === void 0 || runKey(file) === id);
224
+ await Promise.all(files.map((file) => writes.run(runKey(file), async () => {
225
+ if (!unsynced.has(file)) return;
226
+ await syncFile(file);
227
+ unsynced.delete(file);
228
+ })));
229
+ await checkpoints.flush(id);
230
+ }
231
+ async function waitForRunWrites(id) {
232
+ await writes.flush(id);
233
+ }
234
+ const reportedBadLines = /* @__PURE__ */ new Set();
235
+ async function readJsonLines(file, stats) {
236
+ const path = join(RUNS_DIR, file);
237
+ if (!existsSync(path)) return [];
238
+ const out = [];
239
+ let skipped = 0;
240
+ const lines = createInterface({ input: createReadStream(path), crlfDelay: Infinity });
241
+ for await (const line of lines) {
242
+ if (!line.trim()) continue;
243
+ try {
244
+ out.push(JSON.parse(line));
245
+ } catch {
246
+ skipped++;
247
+ }
248
+ }
249
+ if (stats) stats.skipped += skipped;
250
+ if (skipped && !reportedBadLines.has(file)) {
251
+ reportedBadLines.add(file);
252
+ logWarn(`skipped ${skipped} unreadable line${skipped === 1 ? "" : "s"} in a run log`, { file });
253
+ }
254
+ return out;
255
+ }
256
+ async function visitJsonLines(file, visit) {
257
+ const path = join(RUNS_DIR, file);
258
+ if (!existsSync(path)) return;
259
+ const stream = createReadStream(path, { encoding: "utf8", highWaterMark: 64 * 1024 });
260
+ let buffer = "", skipped = 0;
261
+ const parse = (line) => {
262
+ if (!line.trim()) return;
263
+ let value;
264
+ try {
265
+ value = JSON.parse(line);
266
+ } catch {
267
+ skipped++;
268
+ return;
269
+ }
270
+ visit(value);
271
+ };
272
+ try {
273
+ for await (const chunk of stream) {
274
+ buffer += chunk;
275
+ let index;
276
+ while ((index = buffer.indexOf("\n")) >= 0) {
277
+ const line = buffer.slice(0, index);
278
+ buffer = buffer.slice(index + 1);
279
+ if (Buffer.byteLength(line) > 64 * 1024 * 1024) throw new Error(`Run log ${file} contains a record larger than 64 MiB`);
280
+ parse(line);
281
+ }
282
+ if (Buffer.byteLength(buffer) > 64 * 1024 * 1024) throw new Error(`Run log ${file} contains a record larger than 64 MiB`);
283
+ }
284
+ parse(buffer);
285
+ } finally {
286
+ stream.destroy();
287
+ }
288
+ if (skipped && !reportedBadLines.has(file)) {
289
+ reportedBadLines.add(file);
290
+ logWarn(`skipped ${skipped} unreadable line${skipped === 1 ? "" : "s"} in a run log`, { file });
291
+ }
292
+ }
293
+ async function listRunIds() {
294
+ if (!existsSync(RUNS_DIR)) return [];
295
+ const files = await readdir(RUNS_DIR);
296
+ const deleted = new Set(files.filter((f) => f.endsWith(".deleted")).map((f) => f.slice(0, -8)));
297
+ return files.filter((f) => f.endsWith(".jsonl") && !f.endsWith(".journal.jsonl") && !deleted.has(f.slice(0, -6))).map((f) => f.slice(0, -".jsonl".length));
298
+ }
299
+ async function listDeletedRunIds() {
300
+ if (!existsSync(RUNS_DIR)) return [];
301
+ return (await readdir(RUNS_DIR)).filter((file) => /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,199}\.deleted$/.test(file)).map((file) => file.slice(0, -8));
302
+ }
303
+ function deleteRunHistory(id) {
304
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,199}$/.test(id)) throw new Error("Invalid run ID");
305
+ appendBatches.delete(id);
306
+ const work = writes.run(id, async () => {
307
+ await ready;
308
+ await checkpoints.flush(id);
309
+ await writeFile(join(RUNS_DIR, `${id}.deleted`), "", { mode: 384, flush: true });
310
+ for (const suffix of [".jsonl", ".journal.jsonl", ".checkpoint.json", ".cold-index.json", ".cold-detail.json"]) {
311
+ await rm(join(RUNS_DIR, id + suffix), { force: true });
312
+ repaired.delete(id + suffix);
313
+ unsynced.delete(id + suffix);
314
+ directorySynced.delete(join(RUNS_DIR, id + suffix));
315
+ }
316
+ });
317
+ return work;
318
+ }
319
+ async function loadJournal(runId, opts = {}) {
320
+ if (opts.strict) {
321
+ let text;
322
+ try {
323
+ text = await readFile(join(RUNS_DIR, `${runId}.journal.jsonl`), "utf8");
324
+ } catch (err) {
325
+ if (err.code === "ENOENT") return /* @__PURE__ */ new Map();
326
+ throw err;
327
+ }
328
+ const complete = text.slice(0, text.lastIndexOf("\n") + 1);
329
+ const journal = /* @__PURE__ */ new Map();
330
+ for (const [index, line] of complete.split("\n").entries()) {
331
+ if (!line.trim()) continue;
332
+ try {
333
+ const entry = JSON.parse(line);
334
+ if (typeof entry.key !== "string" || !("value" in entry) || (entry.v ?? 1) > JOURNAL_VERSION) throw new Error("invalid or unsupported record");
335
+ journal.set(entry.key, entry.value);
336
+ } catch (err) {
337
+ throw new Error(`Cannot recover journal ${runId}: record ${index + 1} is damaged or uses an unsupported version`, { cause: err });
338
+ }
339
+ }
340
+ return journal;
341
+ }
342
+ const entries = await readJsonLines(`${runId}.journal.jsonl`);
343
+ return new Map(entries.filter((e) => (e.v ?? 1) <= JOURNAL_VERSION).map((e) => [e.key, e.value]));
344
+ }
345
+ function rewriteLines(file, make) {
346
+ appendBatches.delete(runKey(file));
347
+ const done = writes.run(runKey(file), async () => {
348
+ await ready;
349
+ await checkpoints.flush(runKey(file));
350
+ const path = join(RUNS_DIR, file);
351
+ const lines = make(await readJsonLines(file));
352
+ if (!lines) return false;
353
+ const temp = `${path}.${randomBytes(4).toString("hex")}.tmp`;
354
+ try {
355
+ await appendSynced(temp, lines.map((l) => `${JSON.stringify(l)}
356
+ `).join(""));
357
+ await rename(temp, path);
358
+ } catch (err) {
359
+ await unlink(temp).catch(() => {
360
+ });
361
+ throw err;
362
+ }
363
+ repaired.add(file);
364
+ unsynced.delete(file);
365
+ return true;
366
+ });
367
+ return done;
368
+ }
369
+ function journalEntry(runId, key, value) {
370
+ return appendLine(`${runId}.journal.jsonl`, { key, value, v: JOURNAL_VERSION }, { sync: true, strict: true });
371
+ }
372
+ const BOOT_SLACK_MS = 12e4;
373
+ const WRITING_MS = 2e3;
374
+ const bootTime = () => Date.now() - uptime() * 1e3;
375
+ function startTicks(pid) {
376
+ if (process.platform !== "linux") return void 0;
377
+ try {
378
+ const stat2 = readFileSync(`/proc/${pid}/stat`, "utf8");
379
+ const ticks = Number(stat2.slice(stat2.lastIndexOf(")") + 2).split(" ")[19]);
380
+ return Number.isFinite(ticks) ? ticks : void 0;
381
+ } catch {
382
+ return void 0;
383
+ }
384
+ }
385
+ function psStart(pid) {
386
+ return new Promise((resolve) => {
387
+ execFile("ps", ["-o", "lstart=", "-p", String(pid)], { timeout: 5e3, env: childEnv({ LC_ALL: "C" }) }, (err, stdout) => {
388
+ const ms = err ? NaN : Date.parse(String(stdout).trim());
389
+ resolve(Number.isFinite(ms) ? ms : void 0);
390
+ });
391
+ });
392
+ }
393
+ function parseLock(text) {
394
+ const trimmed = text.trim();
395
+ if (/^\d+$/.test(trimmed)) return Number(trimmed) > 0 ? { pid: Number(trimmed), legacy: true } : void 0;
396
+ try {
397
+ const info = JSON.parse(trimmed);
398
+ return Number.isInteger(info?.pid) && info.pid > 0 ? { ...info, legacy: false } : void 0;
399
+ } catch {
400
+ return void 0;
401
+ }
402
+ }
403
+ async function holderAlive(text, mtimeMs) {
404
+ const info = parseLock(text);
405
+ if (!info) return { live: Date.now() - mtimeMs < WRITING_MS };
406
+ const { pid } = info;
407
+ if (pid === process.pid || !alive(pid)) return { live: false, pid };
408
+ if (info.legacy) {
409
+ if (mtimeMs < bootTime() - BOOT_SLACK_MS) return { live: false, pid };
410
+ const started = await psStart(pid);
411
+ return { live: started === void 0 || started <= mtimeMs + WRITING_MS, pid };
412
+ }
413
+ if (info.ticks !== void 0) {
414
+ const ticks = startTicks(pid);
415
+ if (ticks !== void 0) return { live: ticks === info.ticks, pid };
416
+ }
417
+ if (info.started !== void 0) {
418
+ const started = await psStart(pid);
419
+ if (started !== void 0) return { live: Math.abs(started - info.started) < 5e3, pid };
420
+ }
421
+ return { live: info.boot === void 0 || Math.abs(info.boot - bootTime()) < BOOT_SLACK_MS, pid };
422
+ }
423
+ async function readLock(file) {
424
+ let handle;
425
+ try {
426
+ handle = await open(file, "r");
427
+ } catch (err) {
428
+ if (err.code === "ENOENT") return void 0;
429
+ throw new Error(`Can't read ${file}: ${err.message}`);
430
+ }
431
+ try {
432
+ return { stat: await handle.stat(), text: await handle.readFile("utf8") };
433
+ } finally {
434
+ await handle.close();
435
+ }
436
+ }
437
+ async function createLock(temp, file, content) {
438
+ try {
439
+ await link(temp, file);
440
+ return true;
441
+ } catch (err) {
442
+ if (err.code === "EEXIST") return false;
443
+ }
444
+ try {
445
+ await writeFile(file, content, { flag: "wx" });
446
+ return true;
447
+ } catch (err) {
448
+ if (err.code === "EEXIST") return false;
449
+ throw err;
450
+ }
451
+ }
452
+ async function takeOverLock(file, temp, judged) {
453
+ const token = `${file}.takeover`;
454
+ try {
455
+ await link(file, token);
456
+ } catch (err) {
457
+ const code = err.code;
458
+ if (code === "ENOENT") return false;
459
+ if (code === "EEXIST") {
460
+ const age = await stat(token).then((s) => Date.now() - s.ctimeMs, () => 0);
461
+ if (age > 5e3) await unlink(token).catch(() => void 0);
462
+ return false;
463
+ }
464
+ if (await readFile(file, "utf8").catch(() => void 0) === judged.text) await unlink(file).catch(() => void 0);
465
+ return false;
466
+ }
467
+ try {
468
+ const now = await stat(token).catch(() => void 0);
469
+ const text = await readFile(token, "utf8").catch(() => void 0);
470
+ if (!now || now.ino !== judged.stat.ino || now.dev !== judged.stat.dev || text !== judged.text) return false;
471
+ await rename(temp, file);
472
+ return true;
473
+ } finally {
474
+ await unlink(token).catch(() => void 0);
475
+ }
476
+ }
477
+ async function acquireLock(waitMs = 2e4, home = CODEFLOW_HOME) {
478
+ const file = join(home, "server.lock");
479
+ await mkdir(home, { recursive: true });
480
+ const info = { pid: process.pid, started: Math.round(Date.now() - process.uptime() * 1e3), ticks: startTicks(process.pid), boot: Math.round(bootTime()) };
481
+ const content = JSON.stringify(info);
482
+ const temp = `${file}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
483
+ await writeFile(temp, content);
484
+ const deadline = Date.now() + waitMs;
485
+ try {
486
+ for (; ; ) {
487
+ if (await createLock(temp, file, content)) break;
488
+ const seen = await readLock(file);
489
+ if (!seen) continue;
490
+ const holder = await holderAlive(seen.text, seen.stat.mtimeMs);
491
+ if (!holder.live) {
492
+ if (await takeOverLock(file, temp, seen)) break;
493
+ if (Date.now() > deadline) throw new Error(`Couldn't replace the stale ${file}. Delete it and start code-flow again.`);
494
+ await new Promise((r) => setTimeout(r, 25));
495
+ continue;
496
+ }
497
+ if (Date.now() > deadline) {
498
+ throw new Error(`Another code-flow server${holder.pid ? ` (pid ${holder.pid})` : ""} is using ${home}. Stop it, or set CODEFLOW_HOME to a different folder.`);
499
+ }
500
+ await new Promise((r) => setTimeout(r, 250));
501
+ }
502
+ } finally {
503
+ await unlink(temp).catch(() => void 0);
504
+ }
505
+ let released = false;
506
+ const release = () => {
507
+ if (released) return;
508
+ released = true;
509
+ try {
510
+ if (readFileSync(file, "utf8") === content) unlinkSync(file);
511
+ } catch {
512
+ }
513
+ };
514
+ return release;
515
+ }
516
+ function alive(pid) {
517
+ try {
518
+ process.kill(pid, 0);
519
+ return true;
520
+ } catch (err) {
521
+ return err.code === "EPERM";
522
+ }
523
+ }
524
+ export {
525
+ JOURNAL_VERSION,
526
+ RUNS_DIR,
527
+ WRITE_BUFFER_BYTES,
528
+ acquireLock,
529
+ appendLine,
530
+ checkpointHealth,
531
+ deleteRunHistory,
532
+ flushWrites,
533
+ journalEntry,
534
+ listDeletedRunIds,
535
+ listRunIds,
536
+ loadJournal,
537
+ readJsonLines,
538
+ readRunCheckpoint,
539
+ rewriteLines,
540
+ visitJsonLines,
541
+ waitForRunWrites,
542
+ writeHealth,
543
+ writeQueueStats
544
+ };
package/src/stream.js ADDED
@@ -0,0 +1,91 @@
1
+ import { randomBytes } from "node:crypto";
2
+ class TopicError extends Error {
3
+ constructor(status, message) {
4
+ super(message);
5
+ this.status = status;
6
+ }
7
+ }
8
+ const MAX_SUBS = 64;
9
+ class StreamHub {
10
+ constructor(sources) {
11
+ this.sources = sources;
12
+ }
13
+ clients = /* @__PURE__ */ new Map();
14
+ /** A stream opened: `send` writes to it. Returns its client id; call `close` when it ends. */
15
+ open(send, subs) {
16
+ const client = { id: randomBytes(12).toString("hex"), send, subs: /* @__PURE__ */ new Map(), closed: false };
17
+ this.clients.set(client.id, client);
18
+ send({ s: 0, d: { type: "hello", client: client.id } });
19
+ this.update(client.id, { subscribe: subs });
20
+ return client.id;
21
+ }
22
+ close(id) {
23
+ const client = this.clients.get(id);
24
+ if (!client) return;
25
+ client.closed = true;
26
+ this.clients.delete(id);
27
+ for (const sub of client.subs.values()) this.stop(sub);
28
+ client.subs.clear();
29
+ }
30
+ /** Change a stream's subscriptions; false when there's no such stream (it closed). */
31
+ update(id, change) {
32
+ const client = this.clients.get(id);
33
+ if (!client) return false;
34
+ for (const subId of Array.isArray(change.unsubscribe) ? change.unsubscribe : []) {
35
+ const sub = client.subs.get(subId);
36
+ if (!sub) continue;
37
+ client.subs.delete(subId);
38
+ this.stop(sub);
39
+ }
40
+ for (const req of Array.isArray(change.subscribe) ? change.subscribe : []) void this.subscribe(client, req);
41
+ return true;
42
+ }
43
+ /** Open streams and their subscriptions, for tests and health. */
44
+ get stats() {
45
+ return { streams: this.clients.size, subscriptions: [...this.clients.values()].reduce((n, c) => n + c.subs.size, 0) };
46
+ }
47
+ async subscribe(client, req) {
48
+ const { id, topic, after } = req ?? {};
49
+ if (!Number.isInteger(id) || id < 1 || typeof topic !== "string" || topic.length > 200 || client.subs.has(id)) return;
50
+ const fail = (status, message) => client.send({ s: id, d: { type: "error", status, message } });
51
+ if (after !== void 0 && (typeof after !== "string" || after.length > 100)) return fail(400, "That cursor isn't one code-flow gave out");
52
+ if (client.subs.size >= MAX_SUBS) return fail(429, `A window watches at most ${MAX_SUBS} things at once`);
53
+ const colon = topic.indexOf(":");
54
+ const [kind, name] = colon < 0 ? [topic, ""] : [topic.slice(0, colon), topic.slice(colon + 1)];
55
+ const source = Object.hasOwn(this.sources, kind) ? this.sources[kind] : void 0;
56
+ if (!source) return fail(404, `Nothing to watch called \u201C${kind}\u201D`);
57
+ const sub = { topic, dropped: false };
58
+ client.subs.set(id, sub);
59
+ const deliver = (d) => {
60
+ if (!sub.dropped && !client.closed) client.send({ s: id, d });
61
+ };
62
+ try {
63
+ const stop = await source(name, after, deliver);
64
+ if (sub.dropped || client.closed) stop();
65
+ else sub.stop = stop;
66
+ } catch (err) {
67
+ if (client.subs.get(id) === sub) client.subs.delete(id);
68
+ if (sub.dropped || client.closed) return;
69
+ sub.dropped = true;
70
+ if (err instanceof TopicError) fail(err.status, err.message);
71
+ else fail(500, err.message);
72
+ }
73
+ }
74
+ stop(sub) {
75
+ sub.dropped = true;
76
+ try {
77
+ sub.stop?.();
78
+ } catch {
79
+ }
80
+ }
81
+ }
82
+ function cursorSeq(cursor, epoch) {
83
+ const [from, seq] = String(cursor ?? "").split(":");
84
+ const n = Number(seq);
85
+ return from === epoch && seq !== "" && seq !== void 0 && Number.isSafeInteger(n) && n >= 0 ? n : void 0;
86
+ }
87
+ export {
88
+ StreamHub,
89
+ TopicError,
90
+ cursorSeq
91
+ };