@earendil-works/pi-durable 0.99.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/README.md +557 -46
  3. package/dist/entries.d.ts +20 -2
  4. package/dist/entries.d.ts.map +1 -1
  5. package/dist/entries.js +14 -1
  6. package/dist/entries.js.map +1 -1
  7. package/dist/env/index.d.ts +18 -38
  8. package/dist/env/index.d.ts.map +1 -1
  9. package/dist/env/index.js +2 -0
  10. package/dist/env/index.js.map +1 -1
  11. package/dist/env/node.d.ts +2 -0
  12. package/dist/env/node.d.ts.map +1 -1
  13. package/dist/env/node.js +53 -58
  14. package/dist/env/node.js.map +1 -1
  15. package/dist/harness/agent.d.ts +31 -0
  16. package/dist/harness/agent.d.ts.map +1 -0
  17. package/dist/harness/agent.js +209 -0
  18. package/dist/harness/agent.js.map +1 -0
  19. package/dist/harness/compaction.d.ts +56 -0
  20. package/dist/harness/compaction.d.ts.map +1 -0
  21. package/dist/harness/compaction.js +382 -0
  22. package/dist/harness/compaction.js.map +1 -0
  23. package/dist/harness/context.d.ts +8 -0
  24. package/dist/harness/context.d.ts.map +1 -1
  25. package/dist/harness/context.js +32 -21
  26. package/dist/harness/context.js.map +1 -1
  27. package/dist/harness/define.d.ts +18 -0
  28. package/dist/harness/define.d.ts.map +1 -0
  29. package/dist/harness/define.js +25 -0
  30. package/dist/harness/define.js.map +1 -0
  31. package/dist/harness/events.d.ts +166 -0
  32. package/dist/harness/events.d.ts.map +1 -0
  33. package/dist/harness/events.js +319 -0
  34. package/dist/harness/events.js.map +1 -0
  35. package/dist/harness/generation.d.ts +36 -7
  36. package/dist/harness/generation.d.ts.map +1 -1
  37. package/dist/harness/generation.js +356 -82
  38. package/dist/harness/generation.js.map +1 -1
  39. package/dist/harness/harness.d.ts.map +1 -1
  40. package/dist/harness/harness.js +141 -106
  41. package/dist/harness/harness.js.map +1 -1
  42. package/dist/harness/inbox.d.ts +59 -0
  43. package/dist/harness/inbox.d.ts.map +1 -0
  44. package/dist/harness/inbox.js +97 -0
  45. package/dist/harness/inbox.js.map +1 -0
  46. package/dist/harness/json.d.ts +10 -0
  47. package/dist/harness/json.d.ts.map +1 -0
  48. package/dist/harness/json.js +33 -0
  49. package/dist/harness/json.js.map +1 -0
  50. package/dist/harness/live.d.ts +66 -7
  51. package/dist/harness/live.d.ts.map +1 -1
  52. package/dist/harness/live.js +71 -7
  53. package/dist/harness/live.js.map +1 -1
  54. package/dist/harness/output.d.ts +62 -0
  55. package/dist/harness/output.d.ts.map +1 -0
  56. package/dist/harness/output.js +261 -0
  57. package/dist/harness/output.js.map +1 -0
  58. package/dist/harness/prompt.d.ts +9 -6
  59. package/dist/harness/prompt.d.ts.map +1 -1
  60. package/dist/harness/prompt.js +57 -14
  61. package/dist/harness/prompt.js.map +1 -1
  62. package/dist/harness/registry.d.ts +2 -3
  63. package/dist/harness/registry.d.ts.map +1 -1
  64. package/dist/harness/registry.js +69 -277
  65. package/dist/harness/registry.js.map +1 -1
  66. package/dist/harness/scheduler.d.ts +42 -12
  67. package/dist/harness/scheduler.d.ts.map +1 -1
  68. package/dist/harness/scheduler.js +649 -75
  69. package/dist/harness/scheduler.js.map +1 -1
  70. package/dist/harness/submissions.d.ts +14 -9
  71. package/dist/harness/submissions.d.ts.map +1 -1
  72. package/dist/harness/submissions.js +76 -34
  73. package/dist/harness/submissions.js.map +1 -1
  74. package/dist/harness/task-graph.d.ts +49 -0
  75. package/dist/harness/task-graph.d.ts.map +1 -0
  76. package/dist/harness/task-graph.js +163 -0
  77. package/dist/harness/task-graph.js.map +1 -0
  78. package/dist/harness/tool.d.ts +36 -0
  79. package/dist/harness/tool.d.ts.map +1 -0
  80. package/dist/harness/tool.js +415 -0
  81. package/dist/harness/tool.js.map +1 -0
  82. package/dist/harness/types.d.ts +330 -125
  83. package/dist/harness/types.d.ts.map +1 -1
  84. package/dist/harness/types.js.map +1 -1
  85. package/dist/harness/usage.d.ts +18 -0
  86. package/dist/harness/usage.d.ts.map +1 -0
  87. package/dist/harness/usage.js +57 -0
  88. package/dist/harness/usage.js.map +1 -0
  89. package/dist/harness/view.d.ts +43 -0
  90. package/dist/harness/view.d.ts.map +1 -0
  91. package/dist/harness/view.js +189 -0
  92. package/dist/harness/view.js.map +1 -0
  93. package/dist/index.d.ts +13 -5
  94. package/dist/index.d.ts.map +1 -1
  95. package/dist/index.js +8 -2
  96. package/dist/index.js.map +1 -1
  97. package/dist/session/observation.d.ts +18 -11
  98. package/dist/session/observation.d.ts.map +1 -1
  99. package/dist/session/observation.js +24 -13
  100. package/dist/session/observation.js.map +1 -1
  101. package/dist/session/session.d.ts +10 -1
  102. package/dist/session/session.d.ts.map +1 -1
  103. package/dist/session/session.js +23 -5
  104. package/dist/session/session.js.map +1 -1
  105. package/dist/session/transaction.d.ts +12 -3
  106. package/dist/session/transaction.d.ts.map +1 -1
  107. package/dist/session/transaction.js +90 -31
  108. package/dist/session/transaction.js.map +1 -1
  109. package/dist/storage/jsonl/storage.d.ts +22 -2
  110. package/dist/storage/jsonl/storage.d.ts.map +1 -1
  111. package/dist/storage/jsonl/storage.js +1 -1
  112. package/dist/storage/jsonl/storage.js.map +1 -1
  113. package/dist/storage/memory.js +1 -1
  114. package/dist/storage/memory.js.map +1 -1
  115. package/dist/storage/sqlite/database.d.ts +26 -19
  116. package/dist/storage/sqlite/database.d.ts.map +1 -1
  117. package/dist/storage/sqlite/database.js.map +1 -1
  118. package/dist/storage/sqlite/index.d.ts +1 -1
  119. package/dist/storage/sqlite/index.d.ts.map +1 -1
  120. package/dist/storage/sqlite/index.js.map +1 -1
  121. package/dist/storage/sqlite/migrations.js +7 -7
  122. package/dist/storage/sqlite/migrations.js.map +1 -1
  123. package/dist/storage/sqlite/node.d.ts +23 -7
  124. package/dist/storage/sqlite/node.d.ts.map +1 -1
  125. package/dist/storage/sqlite/node.js +129 -47
  126. package/dist/storage/sqlite/node.js.map +1 -1
  127. package/dist/storage/sqlite/storage.d.ts +12 -0
  128. package/dist/storage/sqlite/storage.d.ts.map +1 -1
  129. package/dist/storage/sqlite/storage.js +121 -129
  130. package/dist/storage/sqlite/storage.js.map +1 -1
  131. package/dist/testing/storage-benchmark.d.ts.map +1 -1
  132. package/dist/testing/storage-benchmark.js +0 -1
  133. package/dist/testing/storage-benchmark.js.map +1 -1
  134. package/dist/testing/storage-conformance.d.ts.map +1 -1
  135. package/dist/testing/storage-conformance.js +34 -2
  136. package/dist/testing/storage-conformance.js.map +1 -1
  137. package/dist/tools/bash.d.ts +28 -0
  138. package/dist/tools/bash.d.ts.map +1 -0
  139. package/dist/tools/bash.js +70 -0
  140. package/dist/tools/bash.js.map +1 -0
  141. package/dist/tools/edit-diff.d.ts +89 -0
  142. package/dist/tools/edit-diff.d.ts.map +1 -0
  143. package/dist/tools/edit-diff.js +386 -0
  144. package/dist/tools/edit-diff.js.map +1 -0
  145. package/dist/tools/edit.d.ts +18 -0
  146. package/dist/tools/edit.d.ts.map +1 -0
  147. package/dist/tools/edit.js +117 -0
  148. package/dist/tools/edit.js.map +1 -0
  149. package/dist/tools/env.d.ts +5 -0
  150. package/dist/tools/env.d.ts.map +1 -0
  151. package/dist/tools/env.js +7 -0
  152. package/dist/tools/env.js.map +1 -0
  153. package/dist/tools/file-mutation-queue.d.ts +9 -0
  154. package/dist/tools/file-mutation-queue.d.ts.map +1 -0
  155. package/dist/tools/file-mutation-queue.js +52 -0
  156. package/dist/tools/file-mutation-queue.js.map +1 -0
  157. package/dist/tools/image.d.ts +2 -0
  158. package/dist/tools/image.d.ts.map +1 -0
  159. package/dist/tools/image.js +92 -0
  160. package/dist/tools/image.js.map +1 -0
  161. package/dist/tools/index.d.ts +23 -0
  162. package/dist/tools/index.d.ts.map +1 -0
  163. package/dist/tools/index.js +15 -0
  164. package/dist/tools/index.js.map +1 -0
  165. package/dist/tools/path-utils.d.ts +5 -0
  166. package/dist/tools/path-utils.d.ts.map +1 -0
  167. package/dist/tools/path-utils.js +26 -0
  168. package/dist/tools/path-utils.js.map +1 -0
  169. package/dist/tools/read.d.ts +17 -0
  170. package/dist/tools/read.d.ts.map +1 -0
  171. package/dist/tools/read.js +101 -0
  172. package/dist/tools/read.js.map +1 -0
  173. package/dist/tools/write.d.ts +10 -0
  174. package/dist/tools/write.d.ts.map +1 -0
  175. package/dist/tools/write.js +31 -0
  176. package/dist/tools/write.js.map +1 -0
  177. package/dist/{env/utils/truncate.d.ts → truncate.d.ts} +1 -17
  178. package/dist/truncate.d.ts.map +1 -0
  179. package/dist/truncate.js +146 -0
  180. package/dist/truncate.js.map +1 -0
  181. package/dist/types.d.ts +97 -19
  182. package/dist/types.d.ts.map +1 -1
  183. package/dist/types.js.map +1 -1
  184. package/package.json +32 -14
  185. package/dist/env/utils/adaptive-publisher.d.ts +0 -24
  186. package/dist/env/utils/adaptive-publisher.d.ts.map +0 -1
  187. package/dist/env/utils/adaptive-publisher.js +0 -79
  188. package/dist/env/utils/adaptive-publisher.js.map +0 -1
  189. package/dist/env/utils/output-capture.d.ts +0 -31
  190. package/dist/env/utils/output-capture.d.ts.map +0 -1
  191. package/dist/env/utils/output-capture.js +0 -238
  192. package/dist/env/utils/output-capture.js.map +0 -1
  193. package/dist/env/utils/truncate.d.ts.map +0 -1
  194. package/dist/env/utils/truncate.js +0 -294
  195. package/dist/env/utils/truncate.js.map +0 -1
  196. package/dist/harness/config.d.ts +0 -20
  197. package/dist/harness/config.d.ts.map +0 -1
  198. package/dist/harness/config.js +0 -18
  199. package/dist/harness/config.js.map +0 -1
  200. package/docs/chord-delta-findings.md +0 -358
  201. package/docs/pico-v5-chord-usage.md +0 -373
  202. package/docs/pico-v5-handoff.md +0 -581
  203. package/docs/pico-v5.md +0 -3084
@@ -1,20 +0,0 @@
1
- import type { ModelThinkingLevel } from "@earendil-works/pi-ai";
2
- import type { ConversationRetryPolicy, ConversationStreamOptions } from "./types.ts";
3
- /** Durable per-conversation model, thinking level, request options, and desired tool loadout. */
4
- export type ConversationConfigState = {
5
- model?: {
6
- provider: string;
7
- modelId: string;
8
- };
9
- thinkingLevel: ModelThinkingLevel;
10
- /** Desired tool names in offered order; names may be unregistered in the current process. */
11
- activeTools: string[];
12
- /** Forwarded to every generation request. */
13
- streamOptions?: ConversationStreamOptions;
14
- /** Durable generation attempt retries; absent uses `DEFAULT_RETRY_POLICY`. */
15
- retry?: ConversationRetryPolicy;
16
- };
17
- export declare const DEFAULT_RETRY_POLICY: ConversationRetryPolicy;
18
- /** Built-in configuration document; rewindable so forks start from the configuration at their fork entry. */
19
- export declare const ConversationConfig: import("../types.ts").RewindableConversationDocToken<ConversationConfigState>;
20
- //# sourceMappingURL=config.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/harness/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAEhE,OAAO,KAAK,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAErF,iGAAiG;AACjG,MAAM,MAAM,uBAAuB,GAAG;IACrC,KAAK,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9C,aAAa,EAAE,kBAAkB,CAAC;IAClC,6FAA6F;IAC7F,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,6CAA6C;IAC7C,aAAa,CAAC,EAAE,yBAAyB,CAAC;IAC1C,8EAA8E;IAC9E,KAAK,CAAC,EAAE,uBAAuB,CAAC;CAChC,CAAC;AAEF,eAAO,MAAM,oBAAoB,EAAE,uBAKlC,CAAC;AAEF,6GAA6G;AAC7G,eAAO,MAAM,kBAAkB,+EAQ7B,CAAC"}
@@ -1,18 +0,0 @@
1
- import { defineDoc } from "../documents.js";
2
- export const DEFAULT_RETRY_POLICY = {
3
- enabled: true,
4
- maxRetries: 3,
5
- baseDelayMs: 2000,
6
- maxAgentDelayMs: 60000,
7
- };
8
- /** Built-in configuration document; rewindable so forks start from the configuration at their fork entry. */
9
- export const ConversationConfig = defineDoc({
10
- kind: "pi.conversation.config",
11
- version: 1,
12
- scope: "conversation",
13
- history: "rewindable",
14
- fork: "asOf",
15
- initial: () => ({ thinkingLevel: "off", activeTools: [] }),
16
- checkpointWhen: () => true,
17
- });
18
- //# sourceMappingURL=config.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/harness/config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAe5C,MAAM,CAAC,MAAM,oBAAoB,GAA4B;IAC5D,OAAO,EAAE,IAAI;IACb,UAAU,EAAE,CAAC;IACb,WAAW,EAAE,IAAI;IACjB,eAAe,EAAE,KAAK;CACtB,CAAC;AAEF,6GAA6G;AAC7G,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAA0B;IACpE,IAAI,EAAE,wBAAwB;IAC9B,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,cAAc;IACrB,OAAO,EAAE,YAAY;IACrB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;IAC1D,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI;CAC1B,CAAC,CAAC","sourcesContent":["import type { ModelThinkingLevel } from \"@earendil-works/pi-ai\";\nimport { defineDoc } from \"../documents.ts\";\nimport type { ConversationRetryPolicy, ConversationStreamOptions } from \"./types.ts\";\n\n/** Durable per-conversation model, thinking level, request options, and desired tool loadout. */\nexport type ConversationConfigState = {\n\tmodel?: { provider: string; modelId: string };\n\tthinkingLevel: ModelThinkingLevel;\n\t/** Desired tool names in offered order; names may be unregistered in the current process. */\n\tactiveTools: string[];\n\t/** Forwarded to every generation request. */\n\tstreamOptions?: ConversationStreamOptions;\n\t/** Durable generation attempt retries; absent uses `DEFAULT_RETRY_POLICY`. */\n\tretry?: ConversationRetryPolicy;\n};\n\nexport const DEFAULT_RETRY_POLICY: ConversationRetryPolicy = {\n\tenabled: true,\n\tmaxRetries: 3,\n\tbaseDelayMs: 2000,\n\tmaxAgentDelayMs: 60000,\n};\n\n/** Built-in configuration document; rewindable so forks start from the configuration at their fork entry. */\nexport const ConversationConfig = defineDoc<ConversationConfigState>({\n\tkind: \"pi.conversation.config\",\n\tversion: 1,\n\tscope: \"conversation\",\n\thistory: \"rewindable\",\n\tfork: \"asOf\",\n\tinitial: () => ({ thinkingLevel: \"off\", activeTools: [] }),\n\tcheckpointWhen: () => true,\n});\n"]}
@@ -1,358 +0,0 @@
1
- # Chord delta investigation findings
2
-
3
- ## Decision and scope
4
-
5
- The standalone ID-addressed graph tracker experiment was removed. It solved
6
- reference reassignment and replicated identity, but its memory, import, and
7
- hydration costs were unacceptable for the drawing workload. This rejects the
8
- measured implementation, not every possible graph representation.
9
-
10
- Keep the existing tree/path delta implementation, its weak proxy-cache changes,
11
- and the spread-first cloning improvement. Keeping the weak-cache patch is not a
12
- claim that its performance is satisfactory: permanent retention improved greatly,
13
- but exhaustive reads still allocate gigabytes and became slower.
14
-
15
- The graph prototype never had a package export or Chord service integration.
16
- Removing it does not change State/ReplicatedState or adopt graph semantics in the
17
- Pico specifications. Existing delta alias behavior remains a separate correctness
18
- problem; this investigation did not establish general mutable-graph correctness
19
- for the tree protocol.
20
-
21
- ## Workload and measurement rules
22
-
23
- Measured on an Apple M5 Max with 128 GiB RAM, Node v26.0.0. The document contains
24
- 20,000 strokes with 100 separately allocated `{ x, y, pressure }` point objects
25
- each: two million points, approximately 2.04 million total objects/arrays. It is
26
- not a packed numeric buffer. Full traversal uses a flat strokes array; mutation
27
- benchmarks partition strokes into layers and include a 200 KB string.
28
-
29
- The plain document occupies about **139.5 MiB of JavaScript heap**. Its roughly
30
- 66 MB serialized representation is a different measurement.
31
-
32
- - Use independent processes, three trials per configuration, and rotated variant
33
- order. Report medians. Large graph/traversal workers used an 8 GiB V8 heap limit.
34
- - Separate initial import, snapshot publication, mutation, flush, JSON encoding,
35
- decoding, and replica application. Fixture construction is outside mutation
36
- timing; importing a newly assigned object is inside it.
37
- - Measure producer-only and producer-plus-replica memory separately. Release raw
38
- graph inputs and emitted batches in separate call frames before retained-heap
39
- measurements; otherwise temporary references can pin entire extra documents.
40
- - Ready/retained heap is measured after GC, relative to the warmed empty process.
41
- Sampled heap includes temporary allocations and uncollected garbage. It is not
42
- an exact allocation peak. Maximum RSS includes heap capacity and native/runtime
43
- allocations, not just live JavaScript objects.
44
- - Weak-reference reclamation requires real job/event-loop boundaries, not merely
45
- a synchronous `global.gc()` after a loop. Yield, collect, and let finalizers run.
46
- - Validate serialized replay outside timing. Graph correctness must compare
47
- identity/topology as well as values: deep equality alone misses lost aliases.
48
- - Raw-object and minimal-proxy controls perform no replication. They are local
49
- cost floors, not equivalent implementations.
50
-
51
- The historical measurements below predate the final spread-first clone change
52
- unless explicitly stated. They are not new measurements of the current source.
53
-
54
- ## Operation log versus cloned baseline
55
-
56
- The operation log at `2c995acf4` compares whole-container assignments immediately
57
- and coalesces recorded operations. The baseline design at `86bac52f9` marks dirty
58
- paths, compares against a retained copy at flush, and refreshes that copy.
59
-
60
- A layer swap illustrates the difference. Two independently allocated layers can
61
- have equal geometry and different names. Both designs eventually emit just two
62
- name sets, **73 bytes**, but still inspect the geometry to discover that result.
63
- The baseline additionally refreshes its copy. Tiny output does not imply cheap
64
- mutation or flush.
65
-
66
- For a fair clone-cost comparison, only the baseline's `cloneJson` helper was
67
- replaced with the exact helper from `2c995acf4`. No diff, coalescing, baseline
68
- refresh, or proxy behavior changed. This was the earlier dynamic-assignment
69
- helper, not the later spread-first helper.
70
-
71
- Mutation plus flush, milliseconds:
72
-
73
- | Workload | Original baseline | Matched-clone baseline | Operation log |
74
- | --- | ---: | ---: | ---: |
75
- | One stroke swap | 0.114 | 0.086 | 0.085 |
76
- | Swap 1,000-stroke layers | 89.940 | 47.924 | 21.048 |
77
- | Swap 10,000-stroke layers | 906.390 | 483.264 | 214.425 |
78
- | Ten 1,000-stroke swaps before flush | 90.598 | 49.455 | 209.346 |
79
- | Sixty selections before flush | 0.121 | 0.101 | 2.346 |
80
- | Swap dissimilar 100-stroke layers | 12.097 | 7.515 | 168.273 |
81
-
82
- Batching favors the baseline because it compares the final dirty region once;
83
- the operation log pays assignment-time comparison repeatedly. Ten swaps restore
84
- the original value and emit no operations in all three variants. The dissimilar
85
- case triggers the operation log's 4,096-metadata threshold and publishes a roughly
86
- 66 MB root replacement; the baselines emit about 2 MB of patches. This difference
87
- is not caused by the cloning helper.
88
-
89
- All 63 independent-process configurations completed. Small-fixture checks and
90
- full-size serialized replay checks passed; original and matched-clone baselines
91
- emitted identical batches in the checked cases. RHS references came from
92
- `tracker.target` to model unwrapping. These benchmarks do **not** establish general
93
- alias correctness.
94
-
95
- ### Semantics cannot be inferred from passing performance cases
96
-
97
- Historical probes found replica divergences around aliases, held references,
98
- reindexing, and detach/reinsert, despite the existing tests passing. The baseline
99
- also diverged for shared initial objects and held elements after `unshift`.
100
-
101
- For example, selecting an item by reference and subsequently editing it means
102
- two producer paths observe one object. A tree replica holds two separate copies;
103
- it needs updates at both paths. Keeping a reference across an array insertion
104
- also means subsequent writes must follow the object's new position, not its old
105
- index. Neither cloning a baseline nor caching a proxy at a path automatically
106
- solves these cases.
107
-
108
- Other approaches considered:
109
-
110
- - Deep copy-on-assignment changes alias semantics and adds traversal/allocation.
111
- Earlier clone measurements were about 7.4 ms for 1,000 strokes, 76.6 ms for
112
- 10,000, and 153.8 ms for 20,000. Adding copies increased a large swap from about
113
- 213 ms to 385 ms. Those figures predate spread-first cloning.
114
- - Persistent trees share unchanged structure and avoid a permanent complete
115
- baseline copy. They do not independently solve live aliases or relocation.
116
- - A path-based wire copy operation needs ordering barriers: existing operation
117
- coalescing can otherwise change the source before the copy executes.
118
- - Declared reference fields/application IDs, stricter tree ownership, and explicit
119
- path setters change the contract. No replacement API was selected; ordinary
120
- JavaScript mutation syntax remained the desired interface.
121
-
122
- ## What the graph experiment did
123
-
124
- Each object/array became a canonical node with a numeric ID. Scalars were stored
125
- inline; object edges referenced canonical nodes. A map indexed nodes by ID, a
126
- weak map recognized imported raw objects, and lazy producer proxies exposed
127
- ordinary mutation syntax. This was normalized graph storage, not a reverse-edge
128
- index layered beside a second complete document.
129
-
130
- Wire references used `["@", id]`, with separate object/array definitions and
131
- ID-addressed set, delete, splice, length, and release operations. Revisioned
132
- snapshots allowed recovery. Replicas allocated shells before linking definitions,
133
- materializing actual JavaScript objects with aliases and cycles. Incremental
134
- updates preserved held replica identities; rebasing reset identities.
135
-
136
- Known-object assignments became reference writes. Swapping two huge layers no
137
- longer walked their descendants. Fresh independently reconstructed equal objects
138
- were new identities, however, and needed complete definitions rather than a
139
- small value diff. Detached nodes remained resident until explicit collection.
140
-
141
- Fifty tests covered topology, cycles, held references, array operations, import
142
- rollback, revision recovery, collection, and deterministic mixed mutations. The
143
- full benchmark completed 120 independent processes with value validation and
144
- held-identity checks for graph swaps. The initial full-traversal comparison added
145
- 12 processes. Correctness on these cases did not make the implementation cheap.
146
-
147
- ### Graph mutation and publication costs
148
-
149
- Historical sibling delta before the weak-cache change versus graph, mutation
150
- plus flush:
151
-
152
- | Workload | Tree delta ms | Graph ms | Tree bytes | Graph bytes |
153
- | --- | ---: | ---: | ---: | ---: |
154
- | 100 scattered edits | 0.265 | 0.158 | 5,243 | 2,235 |
155
- | Swap 1,000-stroke layers | 21.523 | 0.002334 | 73 | 111 |
156
- | Swap 10,000-stroke layers | 221.373 | 0.002584 | 73 | 109 |
157
- | Ten 1,000-stroke swaps before flush | 211.370 | 0.015083 | 2 | 471 |
158
- | Sixty selections before flush | 0.768 | 0.046125 | 66 | 102 |
159
- | Append one new 100-point stroke | 0.011 | 0.095 | 3,357 | 7,567 |
160
- | Fresh equal-geometry 1,000-stroke layer | 12.311 | 78.716 | 37 | 7,440,956 |
161
- | Append ten characters to a 200 KB string | 0.010 | 0.000958 | 29 | 200,299 |
162
-
163
- Graph bytes include its envelope; tree bytes are decoded operation arrays, not
164
- path-dictionary encoding. The formats provide different identity guarantees.
165
- The graph retained ordered reference writes for cancelled swaps. Its missing
166
- string append/truncate compression was a prototype limitation, not an inherent
167
- cost of graph addressing.
168
-
169
- With JSON encode/decode and replica application included, the fresh-layer case
170
- cost **165.773 ms** for graph versus **12.017 ms** for tree delta. The 10,000-stroke
171
- swap pipeline cost **0.009958 ms** versus **214.119 ms**. Initial import and
172
- hydration are not included in those incremental numbers.
173
-
174
- Twenty thousand hot nested reads cost about **20.84 ms** for graph, **9.12 ms** for
175
- tree delta, **6.79 ms** for a minimal proxy, and **0.082 ms** for raw objects. This
176
- small working set touched only 1,000 points and missed exhaustive-read allocation.
177
-
178
- ### Initial graph costs and lifetime
179
-
180
- Representative scalar-workload measurements:
181
-
182
- | Metric | Tree delta | Graph |
183
- | --- | ---: | ---: |
184
- | Ready producer heap | 139.5 MiB | 430.4 MiB |
185
- | Ready producer plus replica | 279.3 MiB | 719.3 MiB |
186
- | Initial import | 0.020 ms | 1,118.907 ms |
187
- | Snapshot flush | 169.218 ms | 594.904 ms |
188
- | Snapshot wire size | 66,364,338 bytes | 147,487,101 bytes |
189
- | Snapshot JSON encode | 176.961 ms | 366.942 ms |
190
- | Snapshot JSON decode | 159.388 ms | 1,179.166 ms |
191
- | Snapshot apply | 0.027 ms | 1,417.161 ms |
192
- | Pipeline-process maximum RSS | 852.8 MiB | 4,978.0 MiB |
193
-
194
- Snapshot phase timings came from a separately requested rebase. Tree application
195
- adopts the already decoded snapshot; graph application validates and materializes
196
- objects. RSS includes earlier import/hydration allocations and is not a minimum
197
- live-set requirement.
198
-
199
- Five fresh 1,000-stroke layer replacements grew graph producer heap from **430.2
200
- to 563.9 MiB**. Explicit collection removed 510,010 nodes in **705.5 ms**, emitted
201
- a **7.14 MB** release batch, and left **486.3 MiB** retained. Producer plus replica
202
- went from **719.2 to 967.0 MiB**, then **831.2 MiB** after collection. Lookup-table
203
- capacity and externally held values can prevent a return to the ready footprint.
204
-
205
- Automatic collection, immutable historical snapshots, bounded logs, hardened
206
- transport limits, and service integration were not implemented.
207
-
208
- ## Full traversal exposed the memory problem
209
-
210
- Reading every point through producer proxies allocated wrappers for nearly every
211
- container. Historical retained-heap results, after traversal and GC:
212
-
213
- | Variant | Ready MiB | After full read MiB | Cold read ms | Warm read ms |
214
- | --- | ---: | ---: | ---: | ---: |
215
- | Raw document | 139.47 | 139.49 | 8.00 | 2.95 |
216
- | Minimal cached read-only proxy | 139.47 | 452.50 | 705.10 | 550.35 |
217
- | Tree delta before weak caches | 139.26 | 3,525.93 | 2,720.98 | 470.89 |
218
- | Graph prototype | 430.39 | 2,159.75 | 1,437.65 | 388.74 |
219
- | Historical matched-clone baseline | 294.22 | 2,228.71 | 827.13 | 360.95 |
220
-
221
- The baseline row was measured in a later three-process run using asynchronous GC
222
- boundaries, not extrapolated from its ready heap. Its immediately sampled heap
223
- was **2,245.37 MiB** and maximum RSS **2,456.30 MiB**. No diff was running: this was
224
- read-only traversal. All three historical trackers strongly cached wrappers.
225
-
226
- ### Existing delta: permanent retention fixed, allocation volume not fixed
227
-
228
- A separate matched before/after run measured the weak-cache patch:
229
-
230
- | Metric | Before | After |
231
- | --- | ---: | ---: |
232
- | Ready producer heap | 139.50 MiB | 139.50 MiB |
233
- | Retained after traversal, job boundaries, and GC | 3,526.17 MiB | 203.54 MiB |
234
- | Immediately sampled heap after cold read | 3,531.93 MiB | 2,491.78 MiB |
235
- | Maximum RSS | 3,858.11 MiB | 3,391.06 MiB |
236
- | Cold traversal | 2,397.40 ms | 6,298.76 ms |
237
- | Warm same-job traversal | 466.43 ms | 844.13 ms |
238
-
239
- These are separate runs from the preceding table. Warm time is the median of
240
- three traversals per process after collection: the first reconstructs collected
241
- wrappers and the following two reuse same-job caches.
242
-
243
- The patch separates public proxies from entry/placement metadata, uses weak
244
- ordinary caches and incarnation-safe finalizer cleanup, and keeps explicit alias
245
- metadata separately. Held descendants retain ancestor metadata so array
246
- reindexing still works if ancestor public proxies were collected. Assigned known
247
- proxies are unwrapped, avoiding wrappers embedded in raw targets. Placement reuse
248
- prevents duplicate splice emission after reindexing or reattachment. Shared
249
- handlers, lazy child maps, and job-local caches reduce some overhead.
250
-
251
- But one point read can still allocate a proxy, handler, tracking entry, placement
252
- record, placement set, weak references, cache entries, and finalization records.
253
- Across roughly two million containers, this is gigabytes. `WeakRef` targets newly
254
- created or dereferenced during an uninterrupted synchronous traversal must remain
255
- alive through that job. The temporary strong caches also survive until scheduled
256
- microtask cleanup. Collection afterward does not prevent the allocation peak.
257
-
258
- The remaining roughly **64 MiB** above the raw document after collection is
259
- bookkeeping/index overhead; its exact composition needs a heap profile. The
260
- roughly **1.2 KB per visited container** of additional immediately sampled heap is
261
- an aggregate average, not an allocation-by-allocation attribution.
262
-
263
- Eleven isolated GC regressions cover unused proxy reclamation, tracker disposal,
264
- held identities and descendants, array reacquisition/reindexing, alias rewrapping,
265
- detach/reinsert, root rebasing, blocked views, and repeated wide-root read windows.
266
- Twelve windows over a 10,000-object root array retained about 0.35 MiB bookkeeping.
267
- Normal mutation correctness must not depend on finalizers running promptly.
268
-
269
- This patch does not resolve the general tree-protocol alias problem. Bulk
270
- read-only inspection through `tracker.target` avoids proxy construction, but
271
- writing through it bypasses tracking. That escape hatch does not make normal
272
- producer-proxy traversal cheap.
273
-
274
- ## Clone layout: a separate, smaller improvement
275
-
276
- Why did a 139.5 MiB document plus a baseline occupy 294.2 MiB rather than 279 MiB?
277
- V8 layout inspection on this Node version showed:
278
-
279
- - A literal `{ x, y, pressure }` occupies 48 bytes, with three in-object slots.
280
- - Building `{}` by dynamic assignment occupies 56 bytes, with one unused slot.
281
- - Two million points times eight bytes accounts for **15.26 MiB** extra.
282
-
283
- The old clone therefore occupied about **154.7 MiB**, not 139.5 MiB. The discrepancy
284
- was not hidden proxy allocation.
285
-
286
- Shallow object spread followed by recursive replacement of object-valued children
287
- kept the compact layout. Arrays retain their recursive array branch; null-prototype
288
- objects retain a null prototype. Full-fixture clone-only measurements, three fresh
289
- processes per method:
290
-
291
- | Clone construction | Additional retained heap | Median clone time |
292
- | --- | ---: | ---: |
293
- | Dynamic property assignment | 154.725 MiB | 162.338 ms |
294
- | Spread first, then recurse | 139.466 MiB | 93.924 ms |
295
-
296
- This helper was adopted in existing delta. Regression tests cover detached nested
297
- values, null prototypes, inherited-name collisions, independent expansion of
298
- shared input values, and published payload isolation. V8 layouts/timings are
299
- engine-specific. `structuredClone()` still produced the 56-byte point layout in
300
- the layout probe; JSON round-trip produced 48-byte points but introduces text
301
- encoding/decoding and temporary text allocation.
302
-
303
- A baseline using the new helper would be expected to need about **279 MiB** for
304
- document plus copy. That is an inference from measured clone sizes, not a rerun
305
- of the complete historical baseline. The improvement does not address gigabytes
306
- of proxy bookkeeping during traversal.
307
-
308
- ## Retained reproduction and provenance
309
-
310
- From the repository root:
311
-
312
- ```sh
313
- node --expose-gc packages/chord/test/delta-traversal.bench.ts --quick --out /tmp/delta-traversal-quick.json
314
- node --expose-gc packages/chord/test/delta-traversal.bench.ts --out /tmp/delta-traversal.json
315
- node --expose-gc packages/chord/test/delta-traversal.bench.ts --modes delta --out /tmp/delta-only.json
316
- ```
317
-
318
- The retained benchmark supports raw, minimal-proxy, and current tree-delta modes.
319
- It checks traversal results, uses separate processes, records source hashes, and
320
- allows multiple event-loop/GC turns. It no longer contains the removed graph
321
- implementation or reproduces the historical graph/mutation comparison.
322
-
323
- Targeted tests from `packages/chord`:
324
-
325
- ```sh
326
- node ../../node_modules/vitest/dist/cli.js --run test/delta.test.ts test/delta-clone.test.ts test/delta-tracker/retention.test.ts test/services.test.ts test/service-wire.test.ts test/facets.test.ts
327
- ```
328
-
329
- Historical source identifiers:
330
-
331
- - Baseline: `86bac52f9`; matched clone helper / operation log: `2c995acf4`.
332
- - Pre-retention sibling delta SHA-256:
333
- `af008c77367a0ba362637b3b0422576c752a205339a33eb26479a41ac087957f`.
334
- - Weak-cache delta before spread-first cloning:
335
- `7a51c9295543b6fb47e06d8460f122242ee9a96a0ef5653d5595b8219792a6d8`.
336
- - Removed graph implementation:
337
- `90893bfa02a7aeb0fc1da9fdd62f53027ab3d22728337d00f7040c19344fec6c`.
338
-
339
- The graph code was an uncommitted experiment; its hash identifies the measured
340
- source but does not provide a Git revision from which to restore it. Historical
341
- raw outputs were left in `/tmp/chord-graph-prototype/` (`full.json`,
342
- `traversal-final.json`, `retention-before.json`, `retention-final.json`, and
343
- `baseline-traversal.json`). Matched-baseline data was in
344
- `/tmp/pi-copy-cost.QAlME4/`; clone-only samples in `/tmp/chord-clone-compare.jsonl`.
345
- These are optional local artifacts, not durable dependencies of this document.
346
-
347
- ## Requirements for a future attempt
348
-
349
- Start with exhaustive producer reads, not only small hot working sets and
350
- mutation/flush benchmarks. Measure cold allocation, warm access, cross-job
351
- reclamation, held identities, alias/reindex behavior, import, hydration, and
352
- fresh-object replacement separately. Never present post-GC retained heap as peak
353
- memory, tiny patches as proof of cheap comparison, or successful value replay as
354
- proof of identity correctness.
355
-
356
- No measured design in this investigation simultaneously achieved low retained
357
- and transient memory, cheap ordinary reads, cheap large reference reassignment,
358
- and the desired broad mutable-JavaScript semantics.