@code-yeongyu/senpi 2026.10.25-4 → 2026.10.26-2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/dist/bundle/chunks/{anthropic-messages-WBASBTIW.js → anthropic-messages-ODSFJCXR.js} +3 -3
- package/dist/bundle/chunks/{args-B444OQBK.js → args-SWKJAYQD.js} +1 -1
- package/dist/bundle/chunks/azure-openai-responses-REGXS4XO.js +2 -0
- package/dist/bundle/chunks/bedrock-converse-stream.js +3 -3
- package/dist/bundle/chunks/chunk-2XF6W6F6.js +410 -0
- package/dist/bundle/chunks/{chunk-WDPQ6CZJ.js → chunk-4QVORFQ2.js} +1 -1
- package/dist/bundle/chunks/{chunk-6AOVUTUR.js → chunk-5W3DTM2J.js} +2 -2
- package/dist/bundle/chunks/{chunk-E6WYH3ZL.js → chunk-6GAMRAVM.js} +9 -7
- package/dist/bundle/chunks/{chunk-N7IN46Q5.js → chunk-7IZH2KQX.js} +9 -7
- package/dist/bundle/chunks/{chunk-7GZUFOM3.js → chunk-ARSENVDY.js} +1 -1
- package/dist/bundle/chunks/{chunk-STMXIBXF.js → chunk-CNDPCYNT.js} +1 -1
- package/dist/bundle/chunks/{chunk-XF3LJC3O.js → chunk-CYOHFREV.js} +1 -1
- package/dist/bundle/chunks/{chunk-E6D4RIT7.js → chunk-DNDZPKIQ.js} +2 -2
- package/dist/bundle/chunks/{chunk-E4LBKOZG.js → chunk-EQNUL2WB.js} +4 -4
- package/dist/bundle/chunks/{chunk-2RB5EL4X.js → chunk-ESNSRUW4.js} +1 -1
- package/dist/bundle/chunks/{chunk-4AOMNOGY.js → chunk-FQDARFHT.js} +1 -1
- package/dist/bundle/chunks/{chunk-BWA4FVFW.js → chunk-GPPOPK76.js} +3 -3
- package/dist/bundle/chunks/{chunk-SBGIXMX3.js → chunk-GWGNVYPB.js} +1 -1
- package/dist/bundle/chunks/{chunk-KKD4CYCI.js → chunk-H4WJ4LY3.js} +1 -1
- package/dist/bundle/chunks/{chunk-V2UYXTRR.js → chunk-HWTD4QF3.js} +1 -1
- package/dist/bundle/chunks/chunk-I2WFLKZ2.js +5 -0
- package/dist/bundle/chunks/{chunk-WD6WBYC5.js → chunk-IHKHCWWG.js} +1 -1
- package/dist/bundle/chunks/{chunk-T36LJ6MW.js → chunk-IY4G44MN.js} +1 -1
- package/dist/bundle/chunks/{chunk-BZ7726QO.js → chunk-J2YAYTB2.js} +1 -1
- package/dist/bundle/chunks/{chunk-UFJ2K7Q7.js → chunk-JH7MVRNJ.js} +1 -1
- package/dist/bundle/chunks/chunk-KBSFTXAT.js +2 -0
- package/dist/bundle/chunks/{chunk-OLCDIITS.js → chunk-KRO63T4Q.js} +1 -1
- package/dist/bundle/chunks/{chunk-TTU2RNVX.js → chunk-KTDIIF2F.js} +1 -1
- package/dist/bundle/chunks/{chunk-KNVNKDER.js → chunk-L2O6K3SJ.js} +1 -1
- package/dist/bundle/chunks/{chunk-3BTN337S.js → chunk-NDNIMC5W.js} +1 -1
- package/dist/bundle/chunks/{chunk-ITGYN22E.js → chunk-ODRGPC5S.js} +1 -1
- package/dist/bundle/chunks/{chunk-4MOZXOJX.js → chunk-PDG4XWO2.js} +1 -1
- package/dist/bundle/chunks/{chunk-3QCYBOKN.js → chunk-PPECSSWX.js} +1 -1
- package/dist/bundle/chunks/{chunk-4P7WARQZ.js → chunk-PY4ZQHTN.js} +1 -1
- package/dist/bundle/chunks/{chunk-C2MQ4QXN.js → chunk-RPW3XQW3.js} +48 -42
- package/dist/bundle/chunks/{chunk-JMOUU75H.js → chunk-RQ5JTXUC.js} +1 -1
- package/dist/bundle/chunks/chunk-TQ67PB7U.js +13 -0
- package/dist/bundle/chunks/{chunk-5JJDR6XI.js → chunk-UJPG4ROT.js} +1 -1
- package/dist/bundle/chunks/{chunk-FHXR4ZAC.js → chunk-XM36IFP3.js} +1 -1
- package/dist/bundle/chunks/{chunk-PT6JJUN2.js → chunk-XMNPK472.js} +1 -1
- package/dist/bundle/chunks/{client-WTYEICLH.js → chunk-Y3TMJV3J.js} +3 -3
- package/dist/bundle/chunks/{chunk-GSSD3R5X.js → chunk-Z6BREXV3.js} +1 -1
- package/dist/bundle/chunks/{chunk-KXB5WIS5.js → chunk-Z6SAQ5O3.js} +1 -1
- package/dist/bundle/chunks/{chunk-LOIF6PHI.js → chunk-ZAUVY6ZC.js} +3 -3
- package/dist/bundle/chunks/{chunk-7CNOCJCN.js → chunk-ZP6JJDYV.js} +1 -1
- package/dist/bundle/chunks/cli-main-Q2GA53XM.js +3 -0
- package/dist/bundle/chunks/client-LOS6GSEW.js +2 -0
- package/dist/bundle/chunks/{cloudflare-workers-ai-system-one-K5H6BJP3.js → cloudflare-workers-ai-system-one-WDHT7BBV.js} +1 -1
- package/dist/bundle/chunks/cursor-agent.js +3 -3
- package/dist/bundle/chunks/devin-agent.js +3 -3
- package/dist/bundle/chunks/dispatch-5R3TYAYB.js +9 -0
- package/dist/bundle/chunks/google-generative-ai-FHYFXKLA.js +2 -0
- package/dist/bundle/chunks/google-vertex-5DWE5U5K.js +2 -0
- package/dist/bundle/chunks/{help-fast-path-3Z26DQSV.js → help-fast-path-FETLIF3W.js} +1 -1
- package/dist/bundle/chunks/host-command-XEB7SHT3.js +2 -0
- package/dist/bundle/chunks/{host-lifecycle-3VHHIGDL.js → host-lifecycle-HX3EGRRW.js} +1 -1
- package/dist/bundle/chunks/host-lifecycle.js +2 -2
- package/dist/bundle/chunks/{host-session-command-56XF6ACS.js → host-session-command-FFN52LXJ.js} +1 -1
- package/dist/bundle/chunks/{host-socket-request-QL2WEZTN.js → host-socket-request-K4LCUOUD.js} +1 -1
- package/dist/bundle/chunks/interactive-mode-5DEKY3YW.js +2 -0
- package/dist/bundle/chunks/{mcp-presenter-bootstrap-XUSCQVRC.js → mcp-presenter-bootstrap-FOMKF4TU.js} +1 -1
- package/dist/bundle/chunks/{mistral-conversations-DH44NOYV.js → mistral-conversations-I2PI7DDF.js} +1 -1
- package/dist/bundle/chunks/{multi-session-host-Z2CW4SDM.js → multi-session-host-OPR5VBFO.js} +6 -7
- package/dist/bundle/chunks/{openai-codex-responses-SERAFKAS.js → openai-codex-responses-WSEX6SH7.js} +1 -1
- package/dist/bundle/chunks/{openai-completions-BZB5O45A.js → openai-completions-YCBHTBSS.js} +1 -1
- package/dist/bundle/chunks/openai-responses-GXNSQF4A.js +3 -0
- package/dist/bundle/chunks/{package-manager-cli-N55U6PEO.js → package-manager-cli-W55E3LHP.js} +1 -1
- package/dist/bundle/chunks/{pi-messages-VXATX45B.js → pi-messages-O6GISIRW.js} +2 -2
- package/dist/bundle/chunks/{role-JVSBNKGE.js → role-SHIZQKK4.js} +1 -1
- package/dist/bundle/chunks/{rotation-stream-CBGCBYHV.js → rotation-stream-2PCZWNKF.js} +1 -1
- package/dist/bundle/chunks/rpc-mode-IDUJTSYS.js +2 -0
- package/dist/bundle/chunks/{rpc-monitor-source-AE3C6EQV.js → rpc-monitor-source-DRZXJHTL.js} +1 -1
- package/dist/bundle/chunks/{schedule-command-VS7CZC2J.js → schedule-command-B7RRSANU.js} +1 -1
- package/dist/bundle/chunks/session-control-endpoint-LMZH5JLH.js +3 -0
- package/dist/bundle/chunks/{session-picker-E4KUJDLO.js → session-picker-DTGBOXDR.js} +1 -1
- package/dist/bundle/chunks/session-worker.js +214 -205
- package/dist/bundle/chunks/{typesafe-system-one-P75WJ767.js → typesafe-system-one-XGYJJVSE.js} +1 -1
- package/dist/bundle/cli.js +2 -2
- package/dist/bundle/host-supervisor.js +2 -2
- package/dist/bundle/index.js +1 -1
- package/dist/bundle/rpc-entry.js +1 -1
- package/dist/bundle/runtime-manifest.json +1 -1
- package/dist/cli/host-command-args.d.ts +1 -0
- package/dist/cli/host-command-args.js +5 -0
- package/dist/cli/host-command.js +7 -1
- package/dist/cli.js +21 -3
- package/dist/core/agent-session.d.ts +11 -1
- package/dist/core/agent-session.js +19 -2
- package/dist/core/extensions/builtin/compaction/index.js +7 -1
- package/dist/core/extensions/builtin/herdr/index.js +9 -5
- package/dist/core/extensions/builtin/mcp/auth/oauth-issuer.d.ts +6 -0
- package/dist/core/extensions/builtin/mcp/auth/oauth-issuer.js +25 -0
- package/dist/core/extensions/builtin/mcp/auth/oauth-provider.d.ts +3 -3
- package/dist/core/extensions/builtin/mcp/auth/oauth-provider.js +50 -19
- package/dist/core/extensions/builtin/mcp/auth/oauth-refresh.js +4 -3
- package/dist/core/extensions/builtin/mcp/auth/oauth.js +19 -12
- package/dist/core/extensions/builtin/mcp/catalog-cache.d.ts +2 -1
- package/dist/core/extensions/builtin/mcp/catalog-cache.js +19 -6
- package/dist/core/extensions/builtin/mcp/catalog.d.ts +3 -2
- package/dist/core/extensions/builtin/mcp/catalog.js +20 -3
- package/dist/core/extensions/builtin/mcp/expose/call.d.ts +2 -1
- package/dist/core/extensions/builtin/mcp/expose/call.js +2 -1
- package/dist/core/extensions/builtin/mcp/expose/register.js +1 -1
- package/dist/core/extensions/builtin/mcp/expose/schema-compat.d.ts +2 -2
- package/dist/core/extensions/builtin/mcp/expose/schema-compat.js +11 -1
- package/dist/core/extensions/builtin/mcp/invocation.js +16 -2
- package/dist/core/extensions/builtin/mcp/modern/dispatch.js +1 -0
- package/dist/core/extensions/builtin/mcp/modern/parameter-headers.d.ts +4 -1
- package/dist/core/extensions/builtin/mcp/modern/parameter-headers.js +33 -35
- package/dist/core/extensions/builtin/mcp/modern/request.d.ts +1 -1
- package/dist/core/extensions/builtin/mcp/modern/request.js +4 -1
- package/dist/core/extensions/builtin/mcp/modern/sdk-bridge.js +18 -3
- package/dist/core/extensions/builtin/mcp/modern/stdio-probe.js +23 -0
- package/dist/core/extensions/builtin/mcp/modern/subscriptions.d.ts +14 -0
- package/dist/core/extensions/builtin/mcp/modern/subscriptions.js +226 -0
- package/dist/core/extensions/builtin/mcp/modern/tool-schema.d.ts +13 -0
- package/dist/core/extensions/builtin/mcp/modern/tool-schema.js +75 -0
- package/dist/core/extensions/builtin/mcp/modern/tools.d.ts +107 -0
- package/dist/core/extensions/builtin/mcp/modern/tools.js +53 -0
- package/dist/core/extensions/builtin/mcp/modern/transport.d.ts +2 -0
- package/dist/core/extensions/builtin/mcp/modern/transport.js +4 -2
- package/dist/core/extensions/builtin/mcp/resources.js +4 -0
- package/dist/core/extensions/builtin/mcp/tool-client.d.ts +13 -0
- package/dist/core/extensions/builtin/mcp/tool-client.js +17 -0
- package/dist/core/extensions/builtin/mcp/tool-contracts.d.ts +19 -0
- package/dist/core/extensions/builtin/mcp/tool-contracts.js +2 -0
- package/dist/core/extensions/builtin/mcp/wrap.js +2 -2
- package/dist/core/extensions/builtin/permission-system/bash-command-segments.d.ts +37 -0
- package/dist/core/extensions/builtin/permission-system/bash-command-segments.js +899 -0
- package/dist/core/extensions/builtin/permission-system/parsers.d.ts +2 -0
- package/dist/core/extensions/builtin/permission-system/parsers.js +44 -10
- package/dist/core/extensions/builtin/terminal/extension-state.js +2 -1
- package/dist/core/extensions/builtin/vertex-model-garden/auth.d.ts +9 -0
- package/dist/core/extensions/builtin/vertex-model-garden/auth.js +17 -1
- package/dist/core/extensions/builtin/websearch/websearch/providers/google.d.ts +6 -0
- package/dist/core/extensions/builtin/websearch/websearch/providers/google.js +6 -1
- package/dist/core/extensions/types.d.ts +2 -0
- package/dist/core/lifecycle-children.d.ts +3 -1
- package/dist/core/lifecycle-children.js +79 -15
- package/dist/core/lifecycle-record.js +1 -24
- package/dist/core/lifecycle-session-binding.js +2 -1
- package/dist/core/lifecycle-wait-items.d.ts +17 -0
- package/dist/core/lifecycle-wait-items.js +72 -0
- package/dist/core/model-runtime.js +6 -1
- package/dist/core/provider-api-key-auth.d.ts +6 -1
- package/dist/core/provider-api-key-auth.js +11 -4
- package/dist/core/provider-composer.d.ts +8 -1
- package/dist/core/provider-composer.js +7 -3
- package/dist/core/sdk.js +4 -1
- package/dist/core/session-activity.js +2 -0
- package/dist/core/session-lifecycle-types.d.ts +20 -4
- package/dist/core/session-lifecycle.js +35 -4
- package/dist/core/structured-session-lane.d.ts +33 -0
- package/dist/core/structured-session-lane.js +62 -0
- package/dist/core/work-custody-record.d.ts +2 -1
- package/dist/core/work-custody-record.js +2 -0
- package/dist/core/work-obligations.d.ts +1 -1
- package/dist/core/work-obligations.js +8 -2
- package/dist/modes/interactive/session-control-commands.js +21 -4
- package/dist/modes/interactive/session-control-rpc-commands.d.ts +1 -1
- package/dist/modes/interactive/session-control-rpc-commands.js +19 -0
- package/dist/modes/json-event.d.ts +79 -1
- package/dist/modes/json-event.js +79 -1
- package/dist/modes/rpc/compact-content-index.d.ts +85 -0
- package/dist/modes/rpc/compact-content-index.js +254 -0
- package/dist/modes/rpc/compact-content-store.d.ts +59 -0
- package/dist/modes/rpc/compact-content-store.js +134 -0
- package/dist/modes/rpc/compact-index-pages.d.ts +12 -0
- package/dist/modes/rpc/compact-index-pages.js +31 -0
- package/dist/modes/rpc/compact-session-state.d.ts +83 -0
- package/dist/modes/rpc/compact-session-state.js +197 -0
- package/dist/modes/rpc/compact-spool-dir.d.ts +8 -0
- package/dist/modes/rpc/compact-spool-dir.js +94 -0
- package/dist/modes/rpc/compact-spool-storage.d.ts +53 -0
- package/dist/modes/rpc/compact-spool-storage.js +162 -0
- package/dist/modes/rpc/compact-storage-io.d.ts +28 -0
- package/dist/modes/rpc/compact-storage-io.js +25 -0
- package/dist/modes/rpc/compact-transition.d.ts +41 -0
- package/dist/modes/rpc/compact-transition.js +12 -0
- package/dist/modes/rpc/connection-handler.d.ts +4 -0
- package/dist/modes/rpc/connection-handler.js +51 -13
- package/dist/modes/rpc/connection-question-bridge.d.ts +2 -1
- package/dist/modes/rpc/connection-question-bridge.js +6 -2
- package/dist/modes/rpc/extension-ui-response.d.ts +7 -2
- package/dist/modes/rpc/extension-ui-response.js +27 -7
- package/dist/modes/rpc/get-snapshot.d.ts +18 -0
- package/dist/modes/rpc/get-snapshot.js +43 -0
- package/dist/modes/rpc/host-capabilities.js +3 -0
- package/dist/modes/rpc/host-ensure-start.js +6 -3
- package/dist/modes/rpc/host-ensure-types.d.ts +2 -0
- package/dist/modes/rpc/host-ensure.js +1 -0
- package/dist/modes/rpc/host-handoff.d.ts +2 -0
- package/dist/modes/rpc/host-handover-request.d.ts +1 -1
- package/dist/modes/rpc/host-handover-request.js +4 -3
- package/dist/modes/rpc/host-handover-wire.js +6 -0
- package/dist/modes/rpc/host-idle-handover.d.ts +2 -0
- package/dist/modes/rpc/host-idle-handover.js +3 -1
- package/dist/modes/rpc/host-observe-request.js +1 -0
- package/dist/modes/rpc/host-runner.d.ts +1 -0
- package/dist/modes/rpc/host-runner.js +9 -7
- package/dist/modes/rpc/host-successor.js +4 -0
- package/dist/modes/rpc/json-normal-value.d.ts +6 -0
- package/dist/modes/rpc/json-normal-value.js +100 -0
- package/dist/modes/rpc/mcp-presenter-bootstrap.js +5 -0
- package/dist/modes/rpc/media-placeholders.d.ts +9 -3
- package/dist/modes/rpc/media-placeholders.js +12 -4
- package/dist/modes/rpc/rpc-host-command-types.d.ts +2 -0
- package/dist/modes/rpc/rpc-question-types.d.ts +1 -0
- package/dist/modes/rpc/rpc-read-response-types.d.ts +7 -0
- package/dist/modes/rpc/rpc-session-command-types.d.ts +6 -0
- package/dist/modes/rpc/rpc-session-state-types.d.ts +3 -0
- package/dist/modes/rpc/rpc-session-state.js +2 -0
- package/dist/modes/rpc/rpc-types.d.ts +1 -0
- package/dist/modes/rpc/security-presentations.js +2 -1
- package/dist/modes/rpc/session-attachment-gate.js +1 -0
- package/dist/modes/rpc/session-binding.js +30 -17
- package/dist/modes/rpc/session-command-activity.js +1 -0
- package/dist/modes/rpc/session-event-fanout.d.ts +1 -1
- package/dist/modes/rpc/session-event-fanout.js +17 -9
- package/dist/modes/rpc/session-event-writer.d.ts +2 -0
- package/dist/modes/rpc/session-event-writer.js +24 -5
- package/dist/modes/rpc/session-extension-ui-requests.d.ts +2 -1
- package/dist/modes/rpc/session-extension-ui-requests.js +15 -10
- package/dist/modes/rpc/session-held.js +1 -0
- package/dist/modes/rpc/session-record-stream.d.ts +3 -3
- package/dist/modes/rpc/session-record-stream.js +3 -3
- package/dist/modes/rpc/session-status-stream.d.ts +2 -1
- package/dist/modes/rpc/session-status-stream.js +8 -2
- package/dist/modes/rpc/session-status-subscriptions.d.ts +2 -1
- package/dist/modes/rpc/session-status-subscriptions.js +4 -0
- package/dist/modes/rpc/session-structured-lanes.d.ts +12 -0
- package/dist/modes/rpc/session-structured-lanes.js +24 -0
- package/dist/modes/rpc/session-worker-client.js +6 -5
- package/dist/modes/rpc/session-worker.js +5 -2
- package/dist/modes/rpc/snapshot-types.d.ts +29 -0
- package/dist/modes/rpc/snapshot-types.js +2 -0
- package/dist/modes/rpc/structured-session-records.d.ts +12 -0
- package/dist/modes/rpc/structured-session-records.js +88 -0
- package/dist/modes/rpc/ui-answer-ledger.d.ts +13 -0
- package/dist/modes/rpc/ui-answer-ledger.js +34 -0
- package/dist/senpi +21 -3
- package/docs/extensions.md +7 -1
- package/docs/mcp.md +20 -0
- package/docs/rpc.md +61 -2
- package/package.json +7 -6
- package/dist/bundle/chunks/azure-openai-responses-7I45DI72.js +0 -2
- package/dist/bundle/chunks/chunk-DMI55M2N.js +0 -2
- package/dist/bundle/chunks/chunk-DODH3N6U.js +0 -410
- package/dist/bundle/chunks/chunk-J37GRED2.js +0 -5
- package/dist/bundle/chunks/chunk-TBU6VYHN.js +0 -13
- package/dist/bundle/chunks/cli-main-YYH3IBC6.js +0 -3
- package/dist/bundle/chunks/dispatch-MXG5YWLV.js +0 -9
- package/dist/bundle/chunks/google-generative-ai-UM5Y7T6B.js +0 -2
- package/dist/bundle/chunks/google-vertex-OKU2VQHL.js +0 -2
- package/dist/bundle/chunks/host-command-TDLLHXVI.js +0 -2
- package/dist/bundle/chunks/interactive-mode-OEPULLZK.js +0 -2
- package/dist/bundle/chunks/openai-responses-TNHQ6SJS.js +0 -3
- package/dist/bundle/chunks/rpc-mode-ETXQWEB2.js +0 -2
- package/dist/bundle/chunks/session-control-endpoint-4LEIDIVI.js +0 -3
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import type { CompactContentStore, LeafId } from "./compact-content-store.ts";
|
|
2
|
+
/**
|
|
3
|
+
* A spillable keyed collection: an immutable paged disk base plus one resident override slot per key
|
|
4
|
+
* (senpi#4272). Updates coalesce into that slot, so N updates to one key hold one slot, and a cold row
|
|
5
|
+
* changes without reading its page. Background compaction folds eligible overrides into new pages and
|
|
6
|
+
* publishes them in one synchronous swap that never removes an override newer than the one it read.
|
|
7
|
+
* A failed compaction leaves the previous base in place and the next one free to run.
|
|
8
|
+
* Pages are internal storage only; nothing here is a wire record.
|
|
9
|
+
*/
|
|
10
|
+
type Override<Row> = {
|
|
11
|
+
readonly kind: "row";
|
|
12
|
+
readonly row: Row;
|
|
13
|
+
} | {
|
|
14
|
+
readonly kind: "patch";
|
|
15
|
+
readonly patch: Partial<Row>;
|
|
16
|
+
} | {
|
|
17
|
+
readonly kind: "tombstone";
|
|
18
|
+
};
|
|
19
|
+
interface Page {
|
|
20
|
+
readonly path: string;
|
|
21
|
+
readonly bytes: number;
|
|
22
|
+
/** Published keys; a row skipped at publication stays in the file but is never read back. */
|
|
23
|
+
readonly keys: readonly string[];
|
|
24
|
+
readonly leaves: readonly LeafId[];
|
|
25
|
+
inBase: boolean;
|
|
26
|
+
pins: number;
|
|
27
|
+
}
|
|
28
|
+
interface IndexBase {
|
|
29
|
+
readonly pages: readonly Page[];
|
|
30
|
+
readonly pageOf: ReadonlyMap<string, Page>;
|
|
31
|
+
}
|
|
32
|
+
export interface FrozenIndex<Row> {
|
|
33
|
+
readonly base: IndexBase;
|
|
34
|
+
readonly overrides: ReadonlyMap<string, Override<Row>>;
|
|
35
|
+
}
|
|
36
|
+
export declare class CompactContentIndex<Row extends object> {
|
|
37
|
+
private readonly store;
|
|
38
|
+
private readonly leavesOf;
|
|
39
|
+
private readonly compactAboveRows;
|
|
40
|
+
private base;
|
|
41
|
+
private readonly overrides;
|
|
42
|
+
/** Uncommitted leaves per row override; only overrides absent here are eligible to fold. */
|
|
43
|
+
private readonly unsettledLeaves;
|
|
44
|
+
private readonly leafKey;
|
|
45
|
+
private compacting;
|
|
46
|
+
private tombstones;
|
|
47
|
+
private tombstoneKeyUnits;
|
|
48
|
+
constructor(store: CompactContentStore, leavesOf: (row: Row) => readonly LeafId[], compactAboveRows: number);
|
|
49
|
+
has(key: string): boolean;
|
|
50
|
+
set(key: string, row: Row): void;
|
|
51
|
+
patch(key: string, fields: Partial<Row>): void;
|
|
52
|
+
/** A key only the overlay ever held leaves no tombstone; a committed one is masked until compaction. */
|
|
53
|
+
delete(key: string): void;
|
|
54
|
+
freeze(): FrozenIndex<Row>;
|
|
55
|
+
release(frozen: FrozenIndex<Row>): void;
|
|
56
|
+
/** Every row visible at a frozen root, reading only that root's own pages. */
|
|
57
|
+
entries(frozen: FrozenIndex<Row>): Promise<Map<string, Row>>;
|
|
58
|
+
status(): {
|
|
59
|
+
overrideSlots: number;
|
|
60
|
+
deletionMasks: number;
|
|
61
|
+
deletionMaskBytes: number;
|
|
62
|
+
basePages: number;
|
|
63
|
+
baseRows: number;
|
|
64
|
+
};
|
|
65
|
+
/** The acceptance path checks counts only: an override is eligible once none of its leaves is uncommitted. */
|
|
66
|
+
maybeCompact(): void;
|
|
67
|
+
private committed;
|
|
68
|
+
/** Folds eligible overrides and every small base page; false leaves the previous base untouched. */
|
|
69
|
+
private compact;
|
|
70
|
+
/**
|
|
71
|
+
* The publication point: one synchronous step, so no acceptance observes a half-swapped index. A folded
|
|
72
|
+
* row is published only while its value is still observable (its override is the folded one, or a newer
|
|
73
|
+
* patch over it); a row removed or wholly replaced meanwhile is skipped, so every published leaf is still
|
|
74
|
+
* owned by the folded override or a touched page when its reference is added.
|
|
75
|
+
*/
|
|
76
|
+
private swap;
|
|
77
|
+
private readPage;
|
|
78
|
+
private reclaimPage;
|
|
79
|
+
private put;
|
|
80
|
+
private drop;
|
|
81
|
+
private countTombstone;
|
|
82
|
+
private pinRow;
|
|
83
|
+
}
|
|
84
|
+
export {};
|
|
85
|
+
//# sourceMappingURL=compact-content-index.d.ts.map
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import { isSmallPage, packPages } from "./compact-index-pages.js";
|
|
2
|
+
export class CompactContentIndex {
|
|
3
|
+
constructor(store, leavesOf, compactAboveRows) {
|
|
4
|
+
this.base = { pages: [], pageOf: new Map() };
|
|
5
|
+
this.overrides = new Map();
|
|
6
|
+
/** Uncommitted leaves per row override; only overrides absent here are eligible to fold. */
|
|
7
|
+
this.unsettledLeaves = new Map();
|
|
8
|
+
this.leafKey = new Map();
|
|
9
|
+
this.compacting = false;
|
|
10
|
+
this.tombstones = 0;
|
|
11
|
+
this.tombstoneKeyUnits = 0;
|
|
12
|
+
this.store = store;
|
|
13
|
+
this.leavesOf = leavesOf;
|
|
14
|
+
this.compactAboveRows = compactAboveRows;
|
|
15
|
+
store.onCommitted = (id) => this.committed(id);
|
|
16
|
+
}
|
|
17
|
+
has(key) {
|
|
18
|
+
const override = this.overrides.get(key);
|
|
19
|
+
return override ? override.kind !== "tombstone" : this.base.pageOf.has(key);
|
|
20
|
+
}
|
|
21
|
+
set(key, row) {
|
|
22
|
+
this.put(key, { kind: "row", row });
|
|
23
|
+
}
|
|
24
|
+
patch(key, fields) {
|
|
25
|
+
const current = this.overrides.get(key);
|
|
26
|
+
if (current?.kind === "row")
|
|
27
|
+
this.put(key, { kind: "row", row: { ...current.row, ...fields } });
|
|
28
|
+
else if (current?.kind === "patch")
|
|
29
|
+
this.put(key, { kind: "patch", patch: { ...current.patch, ...fields } });
|
|
30
|
+
else
|
|
31
|
+
this.put(key, { kind: "patch", patch: fields });
|
|
32
|
+
}
|
|
33
|
+
/** A key only the overlay ever held leaves no tombstone; a committed one is masked until compaction. */
|
|
34
|
+
delete(key) {
|
|
35
|
+
if (this.base.pageOf.has(key))
|
|
36
|
+
this.put(key, { kind: "tombstone" });
|
|
37
|
+
else
|
|
38
|
+
this.drop(key);
|
|
39
|
+
}
|
|
40
|
+
freeze() {
|
|
41
|
+
for (const page of this.base.pages)
|
|
42
|
+
page.pins++;
|
|
43
|
+
for (const override of this.overrides.values())
|
|
44
|
+
this.pinRow(override, 1);
|
|
45
|
+
return { base: this.base, overrides: new Map(this.overrides) };
|
|
46
|
+
}
|
|
47
|
+
release(frozen) {
|
|
48
|
+
for (const override of frozen.overrides.values())
|
|
49
|
+
this.pinRow(override, -1);
|
|
50
|
+
for (const page of frozen.base.pages) {
|
|
51
|
+
page.pins--;
|
|
52
|
+
if (!page.inBase && page.pins === 0)
|
|
53
|
+
this.reclaimPage(page, 0, -1);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/** Every row visible at a frozen root, reading only that root's own pages. */
|
|
57
|
+
async entries(frozen) {
|
|
58
|
+
const rows = new Map();
|
|
59
|
+
for (const page of frozen.base.pages)
|
|
60
|
+
for (const [key, row] of await this.readPage(page))
|
|
61
|
+
rows.set(key, row);
|
|
62
|
+
applyOverrides(rows, frozen.overrides);
|
|
63
|
+
return rows;
|
|
64
|
+
}
|
|
65
|
+
status() {
|
|
66
|
+
return {
|
|
67
|
+
overrideSlots: this.overrides.size,
|
|
68
|
+
deletionMasks: this.tombstones,
|
|
69
|
+
deletionMaskBytes: this.tombstoneKeyUnits * 2,
|
|
70
|
+
basePages: this.base.pages.length,
|
|
71
|
+
baseRows: this.base.pageOf.size,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/** The acceptance path checks counts only: an override is eligible once none of its leaves is uncommitted. */
|
|
75
|
+
maybeCompact() {
|
|
76
|
+
if (this.compacting || !this.store.spillAllowed)
|
|
77
|
+
return;
|
|
78
|
+
if (this.overrides.size - this.unsettledLeaves.size < this.compactAboveRows)
|
|
79
|
+
return;
|
|
80
|
+
this.compacting = true;
|
|
81
|
+
this.store.enqueue("page_write", async () => {
|
|
82
|
+
let progressed = false;
|
|
83
|
+
try {
|
|
84
|
+
progressed = await this.compact();
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
this.compacting = false;
|
|
88
|
+
}
|
|
89
|
+
if (progressed)
|
|
90
|
+
this.maybeCompact();
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
committed(id) {
|
|
94
|
+
const key = this.leafKey.get(id);
|
|
95
|
+
if (key === undefined)
|
|
96
|
+
return;
|
|
97
|
+
this.leafKey.delete(id);
|
|
98
|
+
const left = (this.unsettledLeaves.get(key) ?? 0) - 1;
|
|
99
|
+
if (left > 0)
|
|
100
|
+
this.unsettledLeaves.set(key, left);
|
|
101
|
+
else
|
|
102
|
+
this.unsettledLeaves.delete(key);
|
|
103
|
+
this.maybeCompact();
|
|
104
|
+
}
|
|
105
|
+
/** Folds eligible overrides and every small base page; false leaves the previous base untouched. */
|
|
106
|
+
async compact() {
|
|
107
|
+
const base = this.base;
|
|
108
|
+
const selected = [...this.overrides].filter(([key]) => !this.unsettledLeaves.has(key));
|
|
109
|
+
if (selected.length < this.compactAboveRows)
|
|
110
|
+
return false;
|
|
111
|
+
const touched = new Set(base.pages.filter(isSmallPage));
|
|
112
|
+
for (const [key] of selected) {
|
|
113
|
+
const page = base.pageOf.get(key);
|
|
114
|
+
if (page)
|
|
115
|
+
touched.add(page);
|
|
116
|
+
}
|
|
117
|
+
const rows = new Map();
|
|
118
|
+
try {
|
|
119
|
+
for (const page of touched)
|
|
120
|
+
for (const [key, row] of await this.readPage(page))
|
|
121
|
+
rows.set(key, row);
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
applyOverrides(rows, new Map(selected));
|
|
127
|
+
const written = [];
|
|
128
|
+
for (const chunk of packPages([...rows])) {
|
|
129
|
+
const data = Buffer.from(JSON.stringify(chunk));
|
|
130
|
+
const path = await this.store.writeFile(data, "page_write", data.byteLength);
|
|
131
|
+
if (path === undefined || !this.store.spillAllowed) {
|
|
132
|
+
for (const page of written)
|
|
133
|
+
this.store.discard(page.path, page.bytes);
|
|
134
|
+
if (path !== undefined)
|
|
135
|
+
this.store.discard(path, data.byteLength);
|
|
136
|
+
return false;
|
|
137
|
+
}
|
|
138
|
+
written.push({ path, bytes: data.byteLength, rows: chunk });
|
|
139
|
+
}
|
|
140
|
+
this.swap(base, touched, written, new Map(selected));
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* The publication point: one synchronous step, so no acceptance observes a half-swapped index. A folded
|
|
145
|
+
* row is published only while its value is still observable (its override is the folded one, or a newer
|
|
146
|
+
* patch over it); a row removed or wholly replaced meanwhile is skipped, so every published leaf is still
|
|
147
|
+
* owned by the folded override or a touched page when its reference is added.
|
|
148
|
+
*/
|
|
149
|
+
swap(base, touched, written, selected) {
|
|
150
|
+
const pageOf = new Map(base.pageOf);
|
|
151
|
+
for (const page of touched)
|
|
152
|
+
for (const key of page.keys)
|
|
153
|
+
pageOf.delete(key);
|
|
154
|
+
const published = [];
|
|
155
|
+
for (const { path, bytes, rows } of written) {
|
|
156
|
+
const kept = rows.filter(([key]) => {
|
|
157
|
+
const folded = selected.get(key);
|
|
158
|
+
const current = this.overrides.get(key);
|
|
159
|
+
return folded === undefined || current === folded || current?.kind === "patch";
|
|
160
|
+
});
|
|
161
|
+
if (kept.length === 0) {
|
|
162
|
+
this.store.discard(path, bytes);
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
const leaves = kept.flatMap(([, row]) => this.leavesOf(row));
|
|
166
|
+
const page = { path, bytes, keys: kept.map(([key]) => key), leaves, inBase: true, pins: 0 };
|
|
167
|
+
for (const key of page.keys)
|
|
168
|
+
pageOf.set(key, page);
|
|
169
|
+
for (const id of leaves)
|
|
170
|
+
this.store.adjust(id, 1, 0);
|
|
171
|
+
published.push(page);
|
|
172
|
+
}
|
|
173
|
+
this.base = { pages: [...base.pages.filter((page) => !touched.has(page)), ...published], pageOf };
|
|
174
|
+
for (const [key, override] of selected) {
|
|
175
|
+
const current = this.overrides.get(key);
|
|
176
|
+
if (current === override || (current?.kind === "tombstone" && !pageOf.has(key)))
|
|
177
|
+
this.drop(key);
|
|
178
|
+
}
|
|
179
|
+
for (const page of touched) {
|
|
180
|
+
page.inBase = false;
|
|
181
|
+
if (page.pins === 0)
|
|
182
|
+
this.reclaimPage(page, -1, 0);
|
|
183
|
+
else
|
|
184
|
+
for (const id of page.leaves)
|
|
185
|
+
this.store.adjust(id, -1, 1);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
async readPage(page) {
|
|
189
|
+
const data = await this.store.readFile(page.path, page.bytes, "page_read");
|
|
190
|
+
const entries = JSON.parse(data.toString("utf8"));
|
|
191
|
+
if (entries.length === page.keys.length)
|
|
192
|
+
return entries;
|
|
193
|
+
const keys = new Set(page.keys);
|
|
194
|
+
return entries.filter(([key]) => keys.has(key));
|
|
195
|
+
}
|
|
196
|
+
reclaimPage(page, live, pins) {
|
|
197
|
+
for (const id of page.leaves)
|
|
198
|
+
this.store.adjust(id, live, pins);
|
|
199
|
+
this.store.discard(page.path, page.bytes);
|
|
200
|
+
}
|
|
201
|
+
put(key, override) {
|
|
202
|
+
const leaves = override.kind === "row" ? this.leavesOf(override.row) : [];
|
|
203
|
+
for (const id of leaves)
|
|
204
|
+
this.store.adjust(id, 1, 0);
|
|
205
|
+
this.drop(key);
|
|
206
|
+
this.overrides.set(key, override);
|
|
207
|
+
const unsettled = leaves.filter((id) => !this.store.isCommitted(id));
|
|
208
|
+
for (const id of unsettled)
|
|
209
|
+
this.leafKey.set(id, key);
|
|
210
|
+
if (unsettled.length > 0)
|
|
211
|
+
this.unsettledLeaves.set(key, unsettled.length);
|
|
212
|
+
if (override.kind === "tombstone")
|
|
213
|
+
this.countTombstone(key, 1);
|
|
214
|
+
this.maybeCompact();
|
|
215
|
+
}
|
|
216
|
+
drop(key) {
|
|
217
|
+
const old = this.overrides.get(key);
|
|
218
|
+
if (!old)
|
|
219
|
+
return;
|
|
220
|
+
this.overrides.delete(key);
|
|
221
|
+
this.unsettledLeaves.delete(key);
|
|
222
|
+
if (old.kind === "row") {
|
|
223
|
+
for (const id of this.leavesOf(old.row)) {
|
|
224
|
+
this.leafKey.delete(id);
|
|
225
|
+
this.store.adjust(id, -1, 0);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
if (old.kind === "tombstone")
|
|
229
|
+
this.countTombstone(key, -1);
|
|
230
|
+
}
|
|
231
|
+
countTombstone(key, delta) {
|
|
232
|
+
this.tombstones += delta;
|
|
233
|
+
this.tombstoneKeyUnits += delta * key.length;
|
|
234
|
+
}
|
|
235
|
+
pinRow(override, delta) {
|
|
236
|
+
if (override.kind === "row")
|
|
237
|
+
for (const id of this.leavesOf(override.row))
|
|
238
|
+
this.store.adjust(id, 0, delta);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
function applyOverrides(rows, overrides) {
|
|
242
|
+
for (const [key, override] of overrides) {
|
|
243
|
+
if (override.kind === "tombstone")
|
|
244
|
+
rows.delete(key);
|
|
245
|
+
else if (override.kind === "row")
|
|
246
|
+
rows.set(key, override.row);
|
|
247
|
+
else {
|
|
248
|
+
const row = rows.get(key);
|
|
249
|
+
if (row)
|
|
250
|
+
rows.set(key, { ...row, ...override.patch });
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
//# sourceMappingURL=compact-content-index.js.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { CompactSpoolStorage } from "./compact-spool-storage.ts";
|
|
2
|
+
import { type CompactStorageFailure, type CompactStorageIo } from "./compact-storage-io.ts";
|
|
3
|
+
/**
|
|
4
|
+
* Owned bounded leaves for compact current state (senpi#4272, R3.C6.4 / R3.C2.4).
|
|
5
|
+
*
|
|
6
|
+
* Acceptance never waits here: a leaf is created synchronously in RAM, and a spill only relocates
|
|
7
|
+
* that immutable value. Its RAM copy is released only after the whole file was written and closed;
|
|
8
|
+
* a failed write keeps the bytes resident, halts further spilling and is reported through
|
|
9
|
+
* `status()`, never dropped. All storage work runs as one serial asynchronous job chain.
|
|
10
|
+
*/
|
|
11
|
+
/** 32,768 UTF-16 code units (64 KiB as UTF-16LE), so a split surrogate survives a leaf boundary exactly. */
|
|
12
|
+
export declare const COMPACT_LEAF_CODE_UNITS = 32768;
|
|
13
|
+
export type LeafId = number;
|
|
14
|
+
export interface CompactStorageStatus {
|
|
15
|
+
readonly residentPayloadBytes: number;
|
|
16
|
+
readonly pendingIoCopyBytes: number;
|
|
17
|
+
readonly pinnedOnlyBytes: number;
|
|
18
|
+
readonly spilledLiveBytes: number;
|
|
19
|
+
readonly spillBytesWritten: number;
|
|
20
|
+
readonly reclaimDebtBytes: number;
|
|
21
|
+
/** Resident bytes above the spill target that a halted spool keeps in RAM (ruling 08 condition 1). */
|
|
22
|
+
readonly spillFailureRetainedBytes: number;
|
|
23
|
+
readonly spillHalted: boolean;
|
|
24
|
+
readonly failures: readonly CompactStorageFailure[];
|
|
25
|
+
}
|
|
26
|
+
export interface CompactContentStoreOptions {
|
|
27
|
+
/** Private epoch spool directory the store creates itself (0700, never an existing one; files 0600). Not a transcript. */
|
|
28
|
+
readonly dir: string;
|
|
29
|
+
readonly io?: CompactStorageIo;
|
|
30
|
+
/** Spilling starts above this many resident leaf bytes and stops at the target. */
|
|
31
|
+
readonly spillAboveBytes?: number;
|
|
32
|
+
readonly spillTargetBytes?: number;
|
|
33
|
+
}
|
|
34
|
+
export declare class CompactContentStore extends CompactSpoolStorage {
|
|
35
|
+
private readonly spillAboveBytes;
|
|
36
|
+
private readonly spillTargetBytes;
|
|
37
|
+
private readonly leaves;
|
|
38
|
+
/** Resident leaves in creation order, so each spill takes the oldest without rescanning committed ones. */
|
|
39
|
+
private readonly ramLeaves;
|
|
40
|
+
private nextLeafId;
|
|
41
|
+
private residentPayloadBytes;
|
|
42
|
+
private pinnedOnlyBytes;
|
|
43
|
+
private spilledLiveBytes;
|
|
44
|
+
/** Called after a leaf commits, so index compaction can include rows that now reference only disk. */
|
|
45
|
+
onCommitted: ((id: LeafId) => void) | undefined;
|
|
46
|
+
constructor(options: CompactContentStoreOptions);
|
|
47
|
+
/** Takes ownership of an immutable value with one live reference. */
|
|
48
|
+
createLeaf(text: string): LeafId;
|
|
49
|
+
/** Moves live and cut-pin references; the last reference drops RAM or reclaims the file. */
|
|
50
|
+
adjust(id: LeafId, live: number, pins: number): void;
|
|
51
|
+
isCommitted(id: LeafId): boolean;
|
|
52
|
+
/** Reads a leaf the caller still references, from RAM or its committed file. */
|
|
53
|
+
readLeaf(id: LeafId): Promise<string>;
|
|
54
|
+
status(): CompactStorageStatus;
|
|
55
|
+
protected scheduleSpill(): void;
|
|
56
|
+
/** Spills one oldest resident leaf per job, so other storage work interleaves fairly. */
|
|
57
|
+
private spillNext;
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=compact-content-store.d.ts.map
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { CompactSpoolStorage, RECLAIM_WORK_ALLOWANCE_BYTES } from "./compact-spool-storage.js";
|
|
2
|
+
import { nodeCompactStorageIo } from "./compact-storage-io.js";
|
|
3
|
+
/**
|
|
4
|
+
* Owned bounded leaves for compact current state (senpi#4272, R3.C6.4 / R3.C2.4).
|
|
5
|
+
*
|
|
6
|
+
* Acceptance never waits here: a leaf is created synchronously in RAM, and a spill only relocates
|
|
7
|
+
* that immutable value. Its RAM copy is released only after the whole file was written and closed;
|
|
8
|
+
* a failed write keeps the bytes resident, halts further spilling and is reported through
|
|
9
|
+
* `status()`, never dropped. All storage work runs as one serial asynchronous job chain.
|
|
10
|
+
*/
|
|
11
|
+
/** 32,768 UTF-16 code units (64 KiB as UTF-16LE), so a split surrogate survives a leaf boundary exactly. */
|
|
12
|
+
export const COMPACT_LEAF_CODE_UNITS = 32_768;
|
|
13
|
+
export class CompactContentStore extends CompactSpoolStorage {
|
|
14
|
+
constructor(options) {
|
|
15
|
+
super(options.dir, options.io ?? nodeCompactStorageIo);
|
|
16
|
+
this.leaves = new Map();
|
|
17
|
+
/** Resident leaves in creation order, so each spill takes the oldest without rescanning committed ones. */
|
|
18
|
+
this.ramLeaves = new Set();
|
|
19
|
+
this.nextLeafId = 1;
|
|
20
|
+
this.residentPayloadBytes = 0;
|
|
21
|
+
this.pinnedOnlyBytes = 0;
|
|
22
|
+
this.spilledLiveBytes = 0;
|
|
23
|
+
this.spillAboveBytes = options.spillAboveBytes ?? 4 * 1024 * 1024;
|
|
24
|
+
this.spillTargetBytes = options.spillTargetBytes ?? 2 * 1024 * 1024;
|
|
25
|
+
}
|
|
26
|
+
/** Takes ownership of an immutable value with one live reference. */
|
|
27
|
+
createLeaf(text) {
|
|
28
|
+
const id = this.nextLeafId++;
|
|
29
|
+
this.leaves.set(id, { text, units: text.length, live: 1, pins: 0, state: "ram" });
|
|
30
|
+
this.ramLeaves.add(id);
|
|
31
|
+
this.residentPayloadBytes += text.length * 2;
|
|
32
|
+
this.scheduleSpill();
|
|
33
|
+
return id;
|
|
34
|
+
}
|
|
35
|
+
/** Moves live and cut-pin references; the last reference drops RAM or reclaims the file. */
|
|
36
|
+
adjust(id, live, pins) {
|
|
37
|
+
const leaf = this.leaves.get(id);
|
|
38
|
+
if (!leaf)
|
|
39
|
+
throw new Error(`compact leaf ${id} is not owned`);
|
|
40
|
+
const pinnedOnly = leaf.live === 0 && leaf.pins > 0;
|
|
41
|
+
leaf.live += live;
|
|
42
|
+
leaf.pins += pins;
|
|
43
|
+
if (pinnedOnly !== (leaf.live === 0 && leaf.pins > 0))
|
|
44
|
+
this.pinnedOnlyBytes += (pinnedOnly ? -2 : 2) * leaf.units;
|
|
45
|
+
if (leaf.live > 0 || leaf.pins > 0)
|
|
46
|
+
return;
|
|
47
|
+
this.leaves.delete(id);
|
|
48
|
+
this.ramLeaves.delete(id);
|
|
49
|
+
if (leaf.state === "committed" && leaf.path) {
|
|
50
|
+
this.spilledLiveBytes -= leaf.units * 2;
|
|
51
|
+
this.discard(leaf.path, leaf.units * 2);
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
this.residentPayloadBytes -= leaf.units * 2;
|
|
55
|
+
}
|
|
56
|
+
isCommitted(id) {
|
|
57
|
+
return this.leaves.get(id)?.state === "committed";
|
|
58
|
+
}
|
|
59
|
+
/** Reads a leaf the caller still references, from RAM or its committed file. */
|
|
60
|
+
async readLeaf(id) {
|
|
61
|
+
const leaf = this.leaves.get(id);
|
|
62
|
+
if (!leaf)
|
|
63
|
+
throw new Error(`compact leaf ${id} is not owned`);
|
|
64
|
+
if (leaf.text !== undefined)
|
|
65
|
+
return leaf.text;
|
|
66
|
+
return (await this.readFile(leaf.path ?? "", leaf.units * 2, "leaf_read")).toString("utf16le");
|
|
67
|
+
}
|
|
68
|
+
status() {
|
|
69
|
+
return {
|
|
70
|
+
residentPayloadBytes: this.residentPayloadBytes,
|
|
71
|
+
pendingIoCopyBytes: this.pendingIoCopyBytes,
|
|
72
|
+
pinnedOnlyBytes: this.pinnedOnlyBytes,
|
|
73
|
+
spilledLiveBytes: this.spilledLiveBytes,
|
|
74
|
+
spillBytesWritten: this.spillBytesWritten,
|
|
75
|
+
reclaimDebtBytes: this.reclaimDebtBytes,
|
|
76
|
+
spillFailureRetainedBytes: this.spillStopped
|
|
77
|
+
? Math.max(0, this.residentPayloadBytes - this.spillTargetBytes)
|
|
78
|
+
: 0,
|
|
79
|
+
spillHalted: this.spillStopped,
|
|
80
|
+
failures: [...this.failures],
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
scheduleSpill() {
|
|
84
|
+
if (this.spillQueued || !this.spillAllowed || this.residentPayloadBytes <= this.spillAboveBytes)
|
|
85
|
+
return;
|
|
86
|
+
if (this.reclaimDebtBytes > RECLAIM_WORK_ALLOWANCE_BYTES)
|
|
87
|
+
return;
|
|
88
|
+
this.spillQueued = true;
|
|
89
|
+
this.enqueue("spill_write", () => this.spillNext());
|
|
90
|
+
}
|
|
91
|
+
/** Spills one oldest resident leaf per job, so other storage work interleaves fairly. */
|
|
92
|
+
async spillNext() {
|
|
93
|
+
const [id] = this.ramLeaves;
|
|
94
|
+
const leaf = id === undefined ? undefined : this.leaves.get(id);
|
|
95
|
+
if (id === undefined || !leaf || leaf.text === undefined || !this.spillAllowed) {
|
|
96
|
+
this.spillQueued = false;
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
const bytes = leaf.units * 2;
|
|
100
|
+
leaf.state = "pending";
|
|
101
|
+
this.ramLeaves.delete(id);
|
|
102
|
+
let path;
|
|
103
|
+
try {
|
|
104
|
+
path = await this.writeFile(Buffer.from(leaf.text, "utf16le"), "spill_write", bytes);
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
// A failed write, or a failure report that threw, keeps the leaf resident and spillable.
|
|
108
|
+
if (path === undefined) {
|
|
109
|
+
leaf.state = "ram";
|
|
110
|
+
if (this.leaves.get(id) === leaf)
|
|
111
|
+
this.ramLeaves.add(id);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
const owned = this.leaves.get(id) === leaf;
|
|
115
|
+
if (path !== undefined && (!owned || this.retired)) {
|
|
116
|
+
this.discard(path, bytes);
|
|
117
|
+
}
|
|
118
|
+
else if (path !== undefined) {
|
|
119
|
+
leaf.state = "committed";
|
|
120
|
+
leaf.path = path;
|
|
121
|
+
leaf.text = undefined;
|
|
122
|
+
this.residentPayloadBytes -= bytes;
|
|
123
|
+
this.spilledLiveBytes += bytes;
|
|
124
|
+
this.onCommitted?.(id);
|
|
125
|
+
}
|
|
126
|
+
this.spillQueued = false;
|
|
127
|
+
if (this.residentPayloadBytes > this.spillTargetBytes) {
|
|
128
|
+
this.spillQueued = this.spillAllowed;
|
|
129
|
+
if (this.spillQueued)
|
|
130
|
+
this.enqueue("spill_write", () => this.spillNext());
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=compact-content-store.js.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Page layout of the compact settled-row index (senpi#4272): how folded rows are packed into
|
|
3
|
+
* copy-on-write page files, and which base pages are small enough to merge on the next compaction.
|
|
4
|
+
*/
|
|
5
|
+
/** A page under half full in both rows and bytes is rewritten by the next compaction, so pages merge. */
|
|
6
|
+
export declare function isSmallPage(page: {
|
|
7
|
+
readonly keys: readonly string[];
|
|
8
|
+
readonly bytes: number;
|
|
9
|
+
}): boolean;
|
|
10
|
+
/** Packs entries into pages of at most `PAGE_ROWS` rows and `PAGE_BYTES` UTF-8 bytes (one oversized row stands alone). */
|
|
11
|
+
export declare function packPages<Row>(rows: readonly [string, Row][]): [string, Row][][];
|
|
12
|
+
//# sourceMappingURL=compact-index-pages.d.ts.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Page layout of the compact settled-row index (senpi#4272): how folded rows are packed into
|
|
3
|
+
* copy-on-write page files, and which base pages are small enough to merge on the next compaction.
|
|
4
|
+
*/
|
|
5
|
+
const PAGE_ROWS = 128;
|
|
6
|
+
const PAGE_BYTES = 64 * 1024;
|
|
7
|
+
/** A page under half full in both rows and bytes is rewritten by the next compaction, so pages merge. */
|
|
8
|
+
export function isSmallPage(page) {
|
|
9
|
+
return page.keys.length < PAGE_ROWS / 2 && page.bytes < PAGE_BYTES / 2;
|
|
10
|
+
}
|
|
11
|
+
/** Packs entries into pages of at most `PAGE_ROWS` rows and `PAGE_BYTES` UTF-8 bytes (one oversized row stands alone). */
|
|
12
|
+
export function packPages(rows) {
|
|
13
|
+
const pages = [];
|
|
14
|
+
let page = [];
|
|
15
|
+
// UTF-8 bytes of the serialized page: the brackets, plus each entry and its separator.
|
|
16
|
+
let bytes = 1;
|
|
17
|
+
for (const entry of rows) {
|
|
18
|
+
const size = Buffer.byteLength(JSON.stringify(entry)) + 1;
|
|
19
|
+
if (page.length > 0 && (page.length === PAGE_ROWS || bytes + size > PAGE_BYTES)) {
|
|
20
|
+
pages.push(page);
|
|
21
|
+
page = [];
|
|
22
|
+
bytes = 1;
|
|
23
|
+
}
|
|
24
|
+
page.push(entry);
|
|
25
|
+
bytes += size;
|
|
26
|
+
}
|
|
27
|
+
if (page.length > 0)
|
|
28
|
+
pages.push(page);
|
|
29
|
+
return pages;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=compact-index-pages.js.map
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { CompactContentIndex, type FrozenIndex } from "./compact-content-index.ts";
|
|
2
|
+
import { type CompactContentStoreOptions, type CompactStorageStatus, type LeafId } from "./compact-content-store.ts";
|
|
3
|
+
import { type CompactBlockKind, type CompactPersistence, type CompactTransition } from "./compact-transition.ts";
|
|
4
|
+
/**
|
|
5
|
+
* The in-memory current-state owner over compact storage (senpi#4272, R3.C6.4 first half).
|
|
6
|
+
*
|
|
7
|
+
* `accept` is synchronous: it validates a typed transition completely, then publishes the next
|
|
8
|
+
* sequence, or throws and leaves the prior root untouched. Block text lives in owned bounded leaves
|
|
9
|
+
* that spill asynchronously; settled messages live in a spillable index whose persistence status
|
|
10
|
+
* changes only through `message_status`, never through storage success or failure. `freeze` pins
|
|
11
|
+
* an immutable cut synchronously, and `read` materializes it asynchronously from RAM or disk.
|
|
12
|
+
*/
|
|
13
|
+
interface FrozenBlock {
|
|
14
|
+
readonly contentIndex: number;
|
|
15
|
+
readonly kind: CompactBlockKind;
|
|
16
|
+
readonly leaves: readonly LeafId[];
|
|
17
|
+
/** A cut shares the immutable active tail string (under one leaf) instead of minting a leaf per freeze. */
|
|
18
|
+
readonly tail: string;
|
|
19
|
+
}
|
|
20
|
+
interface SettledRow {
|
|
21
|
+
readonly persistence: CompactPersistence;
|
|
22
|
+
readonly blocks: readonly FrozenBlock[];
|
|
23
|
+
}
|
|
24
|
+
interface FrozenCompactState {
|
|
25
|
+
readonly sequence: number;
|
|
26
|
+
readonly active: ReadonlyMap<string, readonly FrozenBlock[]>;
|
|
27
|
+
readonly settled: FrozenIndex<SettledRow>;
|
|
28
|
+
}
|
|
29
|
+
interface CompactMessageView {
|
|
30
|
+
readonly messageId: string;
|
|
31
|
+
readonly persistence: CompactPersistence | undefined;
|
|
32
|
+
readonly blocks: readonly {
|
|
33
|
+
readonly contentIndex: number;
|
|
34
|
+
readonly kind: CompactBlockKind;
|
|
35
|
+
readonly text: string;
|
|
36
|
+
}[];
|
|
37
|
+
}
|
|
38
|
+
export interface CompactSessionStateOptions extends CompactContentStoreOptions {
|
|
39
|
+
/** Index compaction is considered once this many override slots exist (one page by default). */
|
|
40
|
+
readonly compactAboveRows?: number;
|
|
41
|
+
}
|
|
42
|
+
export declare class CompactSessionState {
|
|
43
|
+
private readonly store;
|
|
44
|
+
private readonly settled;
|
|
45
|
+
private readonly active;
|
|
46
|
+
private sequence;
|
|
47
|
+
private frozenTailBytes;
|
|
48
|
+
constructor(options: CompactSessionStateOptions);
|
|
49
|
+
get acceptedSequence(): number;
|
|
50
|
+
accept(transition: CompactTransition): number;
|
|
51
|
+
freeze(): FrozenCompactState;
|
|
52
|
+
release(frozen: FrozenCompactState): void;
|
|
53
|
+
/** Rejects with the storage error when committed content is gone; never returns empty content for it. */
|
|
54
|
+
read(frozen: FrozenCompactState): Promise<{
|
|
55
|
+
sequence: number;
|
|
56
|
+
messages: CompactMessageView[];
|
|
57
|
+
}>;
|
|
58
|
+
storageStatus(): CompactStorageStatus & ReturnType<CompactContentIndex<SettledRow>["status"]> & {
|
|
59
|
+
readonly residentTailBytes: number;
|
|
60
|
+
/** Active tails held by unreleased cuts; may share storage with `residentTailBytes`. */
|
|
61
|
+
readonly frozenTailBytes: number;
|
|
62
|
+
readonly hotWitnessBytes: number;
|
|
63
|
+
};
|
|
64
|
+
idle(): Promise<void>;
|
|
65
|
+
retire(): Promise<void>;
|
|
66
|
+
private start;
|
|
67
|
+
/**
|
|
68
|
+
* Appends only the new suffix. Overlap exists only while a fully covered seed is being replayed
|
|
69
|
+
* (`emitted < covered == units`), so it always lies inside the producer-declared coverage.
|
|
70
|
+
*/
|
|
71
|
+
private delta;
|
|
72
|
+
private replace;
|
|
73
|
+
private setPersistence;
|
|
74
|
+
private remove;
|
|
75
|
+
private block;
|
|
76
|
+
private append;
|
|
77
|
+
/** Settling mints the final tail into a spillable leaf; a freeze of an active block shares the tail. */
|
|
78
|
+
private seal;
|
|
79
|
+
private pin;
|
|
80
|
+
private texts;
|
|
81
|
+
}
|
|
82
|
+
export {};
|
|
83
|
+
//# sourceMappingURL=compact-session-state.d.ts.map
|