@hraness/sponge 0.1.2 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/README.md +164 -17
  2. package/bin/sponge.js +9 -2
  3. package/dist/contracts/index.d.ts +164 -0
  4. package/dist/contracts/index.js +6 -0
  5. package/dist/contracts/validate.d.ts +14 -0
  6. package/dist/contracts/validate.js +187 -0
  7. package/dist/migrations/001.d.ts +3 -0
  8. package/dist/migrations/001.js +33 -0
  9. package/dist/migrations/002.d.ts +3 -0
  10. package/dist/migrations/002.js +6 -0
  11. package/dist/migrations/003.d.ts +3 -0
  12. package/dist/migrations/003.js +10 -0
  13. package/dist/migrations/004.d.ts +3 -0
  14. package/dist/migrations/004.js +9 -0
  15. package/dist/src/acquisition/capture.d.ts +29 -0
  16. package/dist/src/acquisition/capture.js +160 -0
  17. package/dist/src/acquisition/cleanup.d.ts +6 -0
  18. package/dist/src/acquisition/cleanup.js +20 -0
  19. package/dist/src/acquisition/errors.d.ts +3 -0
  20. package/dist/src/acquisition/errors.js +20 -0
  21. package/dist/src/acquisition/files.d.ts +4 -0
  22. package/dist/src/acquisition/files.js +49 -0
  23. package/dist/src/acquisition/index.d.ts +6 -0
  24. package/dist/src/acquisition/index.js +4 -0
  25. package/dist/src/acquisition/lock.d.ts +1 -0
  26. package/dist/src/acquisition/lock.js +2 -0
  27. package/dist/src/acquisition/normalize.d.ts +11 -0
  28. package/dist/src/acquisition/normalize.js +62 -0
  29. package/dist/src/acquisition/pins.d.ts +3 -0
  30. package/dist/src/acquisition/pins.js +9 -0
  31. package/dist/src/acquisition/process.d.ts +16 -0
  32. package/dist/src/acquisition/process.js +63 -0
  33. package/dist/src/acquisition/runtime.d.ts +54 -0
  34. package/dist/src/acquisition/runtime.js +218 -0
  35. package/dist/src/acquisition/verify-helper.d.ts +1 -0
  36. package/dist/src/acquisition/verify-helper.js +19 -0
  37. package/dist/src/app/article.d.ts +23 -0
  38. package/dist/src/app/article.js +82 -0
  39. package/dist/src/app/bundle-files.d.ts +8 -0
  40. package/dist/src/app/bundle-files.js +66 -0
  41. package/dist/src/app/markdown.d.ts +6 -0
  42. package/dist/src/app/markdown.js +90 -0
  43. package/dist/src/app/source.d.ts +15 -0
  44. package/dist/src/app/source.js +40 -0
  45. package/dist/src/cli/arguments.d.ts +9 -0
  46. package/dist/src/cli/arguments.js +106 -0
  47. package/dist/src/cli/config.d.ts +8 -0
  48. package/dist/src/cli/config.js +45 -0
  49. package/dist/src/cli/import.d.ts +4 -0
  50. package/dist/src/cli/import.js +3 -0
  51. package/dist/src/cli/indexing.d.ts +7 -0
  52. package/dist/src/cli/indexing.js +21 -0
  53. package/dist/src/cli/main.d.ts +1 -0
  54. package/dist/src/cli/main.js +333 -0
  55. package/dist/src/cli/portability.d.ts +9 -0
  56. package/dist/src/cli/portability.js +39 -0
  57. package/dist/src/cli/queue.d.ts +2 -0
  58. package/dist/src/cli/queue.js +23 -0
  59. package/dist/src/cli/terminal.d.ts +2 -0
  60. package/dist/src/cli/terminal.js +4 -0
  61. package/dist/src/cli/worker.d.ts +16 -0
  62. package/dist/src/cli/worker.js +62 -0
  63. package/dist/src/importers/apply.d.ts +29 -0
  64. package/dist/src/importers/apply.js +85 -0
  65. package/dist/src/importers/custody.d.ts +6 -0
  66. package/dist/src/importers/custody.js +35 -0
  67. package/dist/src/importers/inspect.d.ts +21 -0
  68. package/dist/src/importers/inspect.js +97 -0
  69. package/dist/src/importers/manifest.d.ts +22 -0
  70. package/dist/src/importers/manifest.js +43 -0
  71. package/dist/src/importers/verify-helper.d.ts +1 -0
  72. package/dist/src/importers/verify-helper.js +27 -0
  73. package/dist/src/indexing/image.d.ts +10 -0
  74. package/dist/src/indexing/image.js +34 -0
  75. package/dist/src/indexing/index.d.ts +53 -0
  76. package/dist/src/indexing/index.js +123 -0
  77. package/dist/src/indexing/jobs.d.ts +14 -0
  78. package/dist/src/indexing/jobs.js +49 -0
  79. package/dist/src/indexing/runner.d.ts +23 -0
  80. package/dist/src/indexing/runner.js +114 -0
  81. package/dist/src/inference/asr/bridge.d.ts +1 -0
  82. package/dist/src/inference/asr/bridge.js +16 -0
  83. package/dist/src/inference/asr/protocol.d.ts +23 -0
  84. package/dist/src/inference/asr/protocol.js +60 -0
  85. package/dist/src/inference/asr/runtime.d.ts +29 -0
  86. package/dist/src/inference/asr/runtime.js +103 -0
  87. package/dist/src/inference/asr/windows.d.ts +9 -0
  88. package/dist/src/inference/asr/windows.js +41 -0
  89. package/dist/src/inference/embedding/index.d.ts +47 -0
  90. package/dist/src/inference/embedding/index.js +122 -0
  91. package/dist/src/media/capture-helper.d.ts +1 -0
  92. package/dist/src/media/capture-helper.js +16 -0
  93. package/dist/src/media/capture.d.ts +32 -0
  94. package/dist/src/media/capture.js +193 -0
  95. package/dist/src/media/handler.d.ts +22 -0
  96. package/dist/src/media/handler.js +82 -0
  97. package/dist/src/media/transcribe.d.ts +29 -0
  98. package/dist/src/media/transcribe.js +93 -0
  99. package/dist/src/metadata/article.d.ts +5 -0
  100. package/dist/src/metadata/article.js +51 -0
  101. package/dist/src/models/artifacts.d.ts +36 -0
  102. package/dist/src/models/artifacts.js +274 -0
  103. package/dist/src/models/catalog.d.ts +35 -0
  104. package/dist/src/models/catalog.js +71 -0
  105. package/dist/src/models/download.d.ts +9 -0
  106. package/dist/src/models/download.js +147 -0
  107. package/dist/src/models/runtime.d.ts +24 -0
  108. package/dist/src/models/runtime.js +37 -0
  109. package/dist/src/pdf/capture-helper.d.ts +1 -0
  110. package/dist/src/pdf/capture-helper.js +12 -0
  111. package/dist/src/pdf/capture.d.ts +29 -0
  112. package/dist/src/pdf/capture.js +156 -0
  113. package/dist/src/pdf/errors.d.ts +2 -0
  114. package/dist/src/pdf/errors.js +13 -0
  115. package/dist/src/pdf/handler.d.ts +24 -0
  116. package/dist/src/pdf/handler.js +84 -0
  117. package/dist/src/pdf/index.d.ts +4 -0
  118. package/dist/src/pdf/index.js +4 -0
  119. package/dist/src/pdf/input.d.ts +2 -0
  120. package/dist/src/pdf/input.js +46 -0
  121. package/dist/src/pdf/normalize.d.ts +10 -0
  122. package/dist/src/pdf/normalize.js +64 -0
  123. package/dist/src/pdf/source-identity.d.ts +4 -0
  124. package/dist/src/pdf/source-identity.js +12 -0
  125. package/dist/src/pdf/source.d.ts +6 -0
  126. package/dist/src/pdf/source.js +22 -0
  127. package/dist/src/pdf/verify-helper.d.ts +1 -0
  128. package/dist/src/pdf/verify-helper.js +17 -0
  129. package/dist/src/portability/backup.d.ts +23 -0
  130. package/dist/src/portability/backup.js +138 -0
  131. package/dist/src/portability/database.d.ts +14 -0
  132. package/dist/src/portability/database.js +78 -0
  133. package/dist/src/portability/export.d.ts +8 -0
  134. package/dist/src/portability/export.js +36 -0
  135. package/dist/src/portability/files.d.ts +33 -0
  136. package/dist/src/portability/files.js +78 -0
  137. package/dist/src/portability/gc.d.ts +21 -0
  138. package/dist/src/portability/gc.js +66 -0
  139. package/dist/src/search/extension.d.ts +3 -0
  140. package/dist/src/search/extension.js +17 -0
  141. package/dist/src/search/hybrid.d.ts +39 -0
  142. package/dist/src/search/hybrid.js +118 -0
  143. package/dist/src/search/query.d.ts +12 -0
  144. package/dist/src/search/query.js +47 -0
  145. package/dist/src/search/vector.d.ts +21 -0
  146. package/dist/src/search/vector.js +21 -0
  147. package/dist/src/service/launch-agent.d.ts +47 -0
  148. package/dist/src/service/launch-agent.js +179 -0
  149. package/dist/src/service/lifetime.d.ts +7 -0
  150. package/dist/src/service/lifetime.js +73 -0
  151. package/dist/src/service/loop.d.ts +21 -0
  152. package/dist/src/service/loop.js +94 -0
  153. package/dist/src/store/files.d.ts +12 -0
  154. package/dist/src/store/files.js +101 -0
  155. package/dist/src/store/index.d.ts +118 -0
  156. package/dist/src/store/index.js +727 -0
  157. package/dist/src/store/types.d.ts +90 -0
  158. package/dist/src/store/types.js +8 -0
  159. package/dist/src/worker/basic.d.ts +10 -0
  160. package/dist/src/worker/basic.js +27 -0
  161. package/dist/src/worker/custody.d.ts +13 -0
  162. package/dist/src/worker/custody.js +185 -0
  163. package/dist/src/worker/engine.d.ts +28 -0
  164. package/dist/src/worker/engine.js +78 -0
  165. package/dist/src/worker/identity.d.ts +7 -0
  166. package/dist/src/worker/identity.js +27 -0
  167. package/dist/src/worker/journal.d.ts +5 -0
  168. package/dist/src/worker/journal.js +30 -0
  169. package/dist/src/worker/supervisor-child.d.ts +1 -0
  170. package/dist/src/worker/supervisor-child.js +33 -0
  171. package/dist/src/worker/supervisor.d.ts +3 -0
  172. package/dist/src/worker/supervisor.js +47 -0
  173. package/native/runtime-manifest.json +40 -0
  174. package/package.json +31 -5
@@ -0,0 +1,118 @@
1
+ import { chunkDocument } from '../indexing/index.js';
2
+ import { pinSearchGeneration, vectorSearch } from './vector.js';
3
+ function lexical(db, query, limit) { const literal = '"' + query.replaceAll('"', '""') + '"'; return db.prepare('SELECT i.id AS itemId,i.title,i.current_revision AS revisionId FROM readable_fts f JOIN items i ON i.id=f.item_id WHERE readable_fts MATCH ? AND i.deleted_at IS NULL AND i.current_revision=f.revision_id ORDER BY rank,i.id LIMIT ?').all(literal, limit); }
4
+ /** Reciprocal rank fusion after per-channel item collapsing; chunks cannot amplify rank. */
5
+ export function fuse(lexicalHits, semanticHits, limit) {
6
+ const hits = new Map();
7
+ for (const [method, rows] of [['lexical', lexicalHits], ['semantic', semanticHits]]) {
8
+ const seen = new Set();
9
+ let rank = 0;
10
+ for (const row of rows) {
11
+ if (seen.has(row.itemId))
12
+ continue;
13
+ seen.add(row.itemId);
14
+ rank++;
15
+ const previous = hits.get(row.itemId), vector = method === 'semantic' ? row : undefined;
16
+ const anchor = vector ? { kind: vector.source.kind === 'image' ? 'image' : 'document', representationId: vector.source.representationId, chunk: vector.source.ordinal, assetSha256: vector.source.assetSha256 } : { kind: 'document' };
17
+ if (previous) {
18
+ previous.score += 1 / (60 + rank);
19
+ previous.methods.push(method);
20
+ if (vector)
21
+ previous.anchor = anchor;
22
+ }
23
+ else
24
+ hits.set(row.itemId, { itemId: row.itemId, revisionId: row.revisionId, title: row.title, score: 1 / (60 + rank), methods: [method], anchor });
25
+ }
26
+ }
27
+ return [...hits.values()].sort((a, b) => b.score - a.score || a.itemId.localeCompare(b.itemId)).slice(0, limit);
28
+ }
29
+ /** Project only producer-authored local PDF/media markers, never remote content. */
30
+ function projectAnchors(db, hits) {
31
+ for (const hit of hits) {
32
+ if (hit.anchor.kind !== 'document' || !hit.anchor.representationId || hit.anchor.chunk === undefined)
33
+ continue;
34
+ const row = db.prepare('SELECT p.markdown,i.kind,r.extractor_name AS extractor FROM representations p JOIN revisions r ON r.id=p.revision_id JOIN items i ON i.id=r.item_id WHERE p.id=? AND p.revision_id=?').get(hit.anchor.representationId, hit.revisionId);
35
+ if (!row || typeof row.markdown !== 'string')
36
+ continue;
37
+ const chunks = chunkDocument(row.markdown), chunk = chunks[hit.anchor.chunk];
38
+ if (chunk === undefined)
39
+ continue;
40
+ // Include the last marker preceding the chunk, including a marker split by chunking.
41
+ let start = 0;
42
+ for (let ordinal = 0; ordinal <= hit.anchor.chunk; ordinal++) {
43
+ start = row.markdown.indexOf(chunks[ordinal], start);
44
+ if (start < 0)
45
+ break;
46
+ if (ordinal < hit.anchor.chunk)
47
+ start += chunks[ordinal].length;
48
+ }
49
+ if (start < 0)
50
+ continue;
51
+ const end = start + chunk.length;
52
+ if (row.kind === 'pdf') {
53
+ const markers = [...row.markdown.matchAll(/^## Page ([1-9][0-9]*)$/gm)];
54
+ const marker = markers.filter(m => m.index <= start).at(-1) ?? markers.find(m => m.index < end);
55
+ if (marker)
56
+ hit.anchor.page = Number(marker[1]);
57
+ }
58
+ else if (row.extractor === 'sponge-media') {
59
+ const markers = [...row.markdown.matchAll(/^\[(\d+):(\d{2}):(\d{2})\] /gm)];
60
+ const marker = markers.filter(m => m.index <= start).at(-1) ?? markers.find(m => m.index < end);
61
+ if (marker)
62
+ hit.anchor.startMs = (Number(marker[1]) * 3600 + Number(marker[2]) * 60 + Number(marker[3])) * 1000;
63
+ }
64
+ }
65
+ }
66
+ export async function hybridSearch(db, query, options = {}) {
67
+ const limit = options.limit ?? 20, timeout = options.semanticTimeoutMs ?? 10000;
68
+ if (!query.trim() || Buffer.byteLength(query) > 4096 || !Number.isSafeInteger(limit) || limit < 1 || limit > 100 || !Number.isSafeInteger(timeout) || timeout < 1 || timeout > 120000)
69
+ throw Error('invalid-search-query');
70
+ const pin = pinSearchGeneration(db, options.profileId ?? 'default'), warnings = [];
71
+ let encoded;
72
+ if (pin && options.embed) {
73
+ const controller = new AbortController(), abort = () => controller.abort();
74
+ options.signal?.addEventListener('abort', abort, { once: true });
75
+ if (options.signal?.aborted)
76
+ abort();
77
+ let timedOut = false;
78
+ const timer = setTimeout(() => { timedOut = true; controller.abort(); }, timeout);
79
+ try {
80
+ const result = await options.embed(query, pin, controller.signal);
81
+ if (timedOut)
82
+ warnings.push('semantic-timeout');
83
+ else if (controller.signal.aborted)
84
+ warnings.push('semantic-unavailable');
85
+ else
86
+ encoded = result;
87
+ }
88
+ catch {
89
+ warnings.push(timedOut ? 'semantic-timeout' : 'semantic-unavailable');
90
+ }
91
+ finally {
92
+ clearTimeout(timer);
93
+ options.signal?.removeEventListener('abort', abort);
94
+ }
95
+ }
96
+ else
97
+ warnings.push('semantic-unavailable');
98
+ db.exec('BEGIN');
99
+ try {
100
+ let semantic = [];
101
+ if (pin && encoded) {
102
+ try {
103
+ semantic = vectorSearch(db, pin, encoded, Math.min(500, limit * 5));
104
+ }
105
+ catch {
106
+ warnings.push('semantic-unavailable');
107
+ }
108
+ }
109
+ const result = { hits: fuse(lexical(db, query, Math.min(500, limit * 5)), semantic, limit), warnings, generation: pin };
110
+ projectAnchors(db, result.hits);
111
+ db.exec('COMMIT');
112
+ return result;
113
+ }
114
+ catch (error) {
115
+ db.exec('ROLLBACK');
116
+ throw error;
117
+ }
118
+ }
@@ -0,0 +1,12 @@
1
+ import type { LibraryStore } from '../store/index.js';
2
+ import { embedBatch, type EmbeddingRuntime } from '../inference/embedding/index.js';
3
+ import type { QueryEmbedder } from './hybrid.js';
4
+ /** Queries mutate coordination journals only, never library content or job tables. */
5
+ export declare function createQueryEmbedder(store: LibraryStore, runtime: EmbeddingRuntime, options?: {
6
+ embed?: typeof embedBatch;
7
+ onReady?: (context: {
8
+ attemptId: string;
9
+ ownerDirectory: string;
10
+ scratchRoot: string;
11
+ }) => Promise<void>;
12
+ }): QueryEmbedder;
@@ -0,0 +1,47 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { join } from 'node:path';
3
+ import { rm } from 'node:fs/promises';
4
+ import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
5
+ import { directory, writeJournal, readJournal, syncDir } from '../worker/journal.js';
6
+ import { supervisedRunner } from '../worker/supervisor.js';
7
+ import { embedBatch, embeddingFingerprint, encodeEmbeddingRequest, EmbeddingError } from '../inference/embedding/index.js';
8
+ /** Queries mutate coordination journals only, never library content or job tables. */
9
+ export function createQueryEmbedder(store, runtime, options = {}) {
10
+ return async (query, pin, signal) => {
11
+ if (signal.aborted)
12
+ throw new EmbeddingError('cancelled');
13
+ if (pin.fingerprint !== embeddingFingerprint(runtime.helperSha256))
14
+ throw new EmbeddingError('integrity');
15
+ const custody = await acquireWorkerCustody(store.root), attemptId = randomUUID();
16
+ let scratchRoot, cleanupUnknown = false;
17
+ try {
18
+ const attempts = await directory(join(custody.directory, 'attempts', attemptId));
19
+ await writeJournal(join(attempts, 'query.json'), { schemaVersion: 1, kind: 'query', attemptId, ownerIncarnation: custody.incarnation });
20
+ scratchRoot = await directory(join(store.root, 'staging', 'query-' + attemptId));
21
+ await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, kind: 'query', requestId: attemptId, ownerIncarnation: custody.incarnation });
22
+ await options.onReady?.({ attemptId, ownerDirectory: custody.directory, scratchRoot });
23
+ const inputs = [{ kind: 'query', text: query }], expected = encodeEmbeddingRequest(inputs).inputHashes[0];
24
+ const result = await (options.embed ?? embedBatch)(inputs, { runtime, scratchRoot, signal, runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId) });
25
+ if (result.fingerprint !== pin.fingerprint || result.inputHashes.length !== 1 || result.inputHashes[0] !== expected || result.vectors.length !== 1)
26
+ throw new EmbeddingError('integrity');
27
+ return { fingerprint: result.fingerprint, vector: result.vectors[0] };
28
+ }
29
+ catch (error) {
30
+ cleanupUnknown = error.code === 'cleanup-required';
31
+ throw error;
32
+ }
33
+ finally {
34
+ const proved = !cleanupUnknown && await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId);
35
+ if (!proved)
36
+ throw new EmbeddingError('cleanup-required');
37
+ if (scratchRoot) {
38
+ const marker = await readJournal(join(scratchRoot, 'ownership.json'));
39
+ if (marker.schemaVersion !== 1 || marker.kind !== 'query' || marker.requestId !== attemptId || marker.ownerIncarnation !== custody.incarnation)
40
+ throw new EmbeddingError('cleanup-required');
41
+ await rm(scratchRoot, { recursive: true });
42
+ await syncDir(join(store.root, 'staging'));
43
+ }
44
+ await custody.release();
45
+ }
46
+ };
47
+ }
@@ -0,0 +1,21 @@
1
+ import type { DatabaseSync } from 'node:sqlite';
2
+ import type { IndexInput } from '../indexing/index.js';
3
+ export interface SearchGeneration {
4
+ id: string;
5
+ fingerprint: string;
6
+ dimensions: number;
7
+ profileId: string;
8
+ }
9
+ export interface VectorHit {
10
+ itemId: string;
11
+ revisionId: string;
12
+ title: string;
13
+ source: IndexInput;
14
+ distance: number;
15
+ }
16
+ export declare function pinSearchGeneration(db: DatabaseSync, profileId: string): SearchGeneration | undefined;
17
+ /** Native scalar scan keeps vectors inside SQLite; SQL bounds materialized result rows. */
18
+ export declare function vectorSearch(db: DatabaseSync, pin: SearchGeneration, query: {
19
+ fingerprint: string;
20
+ vector: readonly number[];
21
+ }, limit?: number): VectorHit[];
@@ -0,0 +1,21 @@
1
+ export function pinSearchGeneration(db, profileId) {
2
+ const row = db.prepare('SELECT g.id,g.fingerprint,s.dimensions FROM active_indexes a JOIN index_generations g ON g.id=a.generation_id JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE a.profile_id=?').get(profileId);
3
+ if (!row)
4
+ return undefined;
5
+ return { id: String(row.id), fingerprint: String(row.fingerprint), dimensions: Number(row.dimensions), profileId };
6
+ }
7
+ /** Native scalar scan keeps vectors inside SQLite; SQL bounds materialized result rows. */
8
+ export function vectorSearch(db, pin, query, limit = 50) {
9
+ if (!Number.isSafeInteger(limit) || limit < 1 || limit > 500 || query.fingerprint !== pin.fingerprint || query.vector.length !== pin.dimensions || query.vector.some(n => !Number.isFinite(n)) || Math.abs(Math.hypot(...query.vector) - 1) > .001)
10
+ throw Error('invalid-search-vector');
11
+ if (db.prepare('SELECT vec_version() AS version').get()?.version !== 'v0.1.9')
12
+ throw Error('unsupported-vector-runtime');
13
+ const generation = db.prepare('SELECT fingerprint FROM index_generations WHERE id=?').get(pin.id);
14
+ if (!generation || generation.fingerprint !== pin.fingerprint)
15
+ throw Error('search-generation-missing');
16
+ const bytes = Buffer.alloc(query.vector.length * 4);
17
+ query.vector.forEach((v, i) => bytes.writeFloatLE(v, i * 4));
18
+ const rows = db.prepare('SELECT v.source_id,v.revision_id,v.derivation_hash,v.asset_sha256,i.id AS item_id,i.title,vec_distance_cosine(v.vector,?) AS distance FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY distance,v.source_id LIMIT ?').all(bytes, pin.id, limit);
19
+ return rows.map(row => { const descriptor = JSON.parse(String(row.source_id)), distance = Number(row.distance); if (!Array.isArray(descriptor) || descriptor[0] !== row.revision_id || !['document', 'image'].includes(String(descriptor[1])) || typeof descriptor[2] !== 'string' || !Number.isFinite(distance))
20
+ throw Error('invalid-search-index'); const source = { sourceId: String(row.source_id), itemId: String(row.item_id), revisionId: String(row.revision_id), derivationHash: String(row.derivation_hash), kind: descriptor[1], ...(descriptor[1] === 'image' ? { assetSha256: String(row.asset_sha256) } : { representationId: descriptor[2], ordinal: Number(descriptor[3]) }) }; return { itemId: String(row.item_id), revisionId: String(row.revision_id), title: String(row.title), source, distance }; });
21
+ }
@@ -0,0 +1,47 @@
1
+ import type { LibraryStore } from '../store/index.js';
2
+ import { type ProcessIdentity } from '../worker/identity.js';
3
+ export interface LaunchResult {
4
+ code: number;
5
+ stdout: string;
6
+ }
7
+ export interface ServiceEnvironment {
8
+ platform: NodeJS.Platform;
9
+ uid: number;
10
+ home: string;
11
+ node: string;
12
+ cli: string;
13
+ version: string;
14
+ launchctl: (args: string[]) => Promise<LaunchResult>;
15
+ alive: (identity: ProcessIdentity) => Promise<boolean>;
16
+ pause: (ms: number) => Promise<void>;
17
+ }
18
+ export declare function serviceEnvironment(version: string): ServiceEnvironment;
19
+ interface Pin {
20
+ path: string;
21
+ bytes: number;
22
+ sha256: string;
23
+ }
24
+ interface Installation {
25
+ language: "en" | null;
26
+ schemaVersion: 1;
27
+ id: string;
28
+ label: string;
29
+ uid: number;
30
+ version: string;
31
+ node: string;
32
+ cli: string;
33
+ library: string;
34
+ plist: string;
35
+ plistHash: string;
36
+ launcher: string;
37
+ launcherHash: string;
38
+ pins: Pin[];
39
+ }
40
+ export declare function agentPlist(record: Pick<Installation, 'label' | 'node' | 'launcher'>): string;
41
+ export declare function serviceStatus(store: LibraryStore, env: ServiceEnvironment): Promise<Record<string, unknown>>;
42
+ export declare function installService(store: LibraryStore, env: ServiceEnvironment, options?: {
43
+ language?: 'en';
44
+ }): Promise<Record<string, unknown>>;
45
+ export declare function uninstallService(store: LibraryStore, env: ServiceEnvironment): Promise<Record<string, unknown>>;
46
+ export declare function installedServiceId(store: LibraryStore, env: ServiceEnvironment): string;
47
+ export {};
@@ -0,0 +1,179 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { promisify } from 'node:util';
3
+ import { homedir } from 'node:os';
4
+ import { join, dirname, resolve } from 'node:path';
5
+ import { randomUUID } from 'node:crypto';
6
+ import { realpathSync, readdirSync, lstatSync, readFileSync, existsSync, unlinkSync } from 'node:fs';
7
+ import { StoreError } from '../store/types.js';
8
+ import { digest, safeDirectory, regularFile, syncDirectory } from '../store/files.js';
9
+ import { describe, exact, readExact, openRead, writePrivateExclusive } from '../portability/files.js';
10
+ import { fstatSync, closeSync } from 'node:fs';
11
+ import { identityAlive } from '../worker/identity.js';
12
+ import { recoverServiceLifetime } from './lifetime.js';
13
+ const exec = promisify(execFile);
14
+ export function serviceEnvironment(version) { return { platform: process.platform, uid: process.getuid?.() ?? -1, home: realpathSync(homedir()), node: realpathSync(process.execPath), cli: realpathSync(process.argv[1]), version, launchctl: async (args) => { try {
15
+ const r = await exec('/bin/launchctl', args, { timeout: 30000, maxBuffer: 65536, env: { HOME: homedir(), PATH: '/usr/bin:/bin', LANG: 'en_US.UTF-8' } });
16
+ return { code: 0, stdout: r.stdout };
17
+ }
18
+ catch (error) {
19
+ const e = error;
20
+ if (typeof e.code === 'number')
21
+ return { code: e.code, stdout: e.stdout ?? '' };
22
+ throw new StoreError('unavailable');
23
+ } }, alive: identityAlive, pause: ms => new Promise(r => setTimeout(r, ms)) }; }
24
+ function paths(store, env) { if (env.platform !== 'darwin')
25
+ throw new StoreError('unavailable'); if (!Number.isSafeInteger(env.uid) || env.uid <= 0)
26
+ throw new StoreError('invalid-input'); const root = join(store.root, 'runtime', 'service'), agents = join(env.home, 'Library', 'LaunchAgents'); const label = 'computer.sponge.worker.' + digest(store.root).slice(0, 20); return { root, agents, label, record: join(root, 'installation.json'), plist: join(agents, label + '.plist'), target: `gui/${env.uid}/${label}` }; }
27
+ function readJson(path) { const fd = openRead(path); try {
28
+ const size = fstatSync(fd).size;
29
+ if (size > 4 * 1024 * 1024)
30
+ throw new StoreError('limit-exceeded');
31
+ return JSON.parse(readExact(fd, size).toString());
32
+ }
33
+ finally {
34
+ closeSync(fd);
35
+ } }
36
+ function installation(store, env) {
37
+ const p = paths(store, env);
38
+ if (!existsSync(p.record))
39
+ return null;
40
+ const r = exact(readJson(p.record), ['language', 'schemaVersion', 'id', 'label', 'uid', 'version', 'node', 'cli', 'library', 'plist', 'plistHash', 'launcher', 'launcherHash', 'pins']);
41
+ if ((r.language !== null && r.language !== 'en') || r.schemaVersion !== 1 || typeof r.id !== 'string' || !/^[a-f0-9-]{36}$/u.test(r.id) || r.label !== p.label || r.uid !== env.uid || r.library !== store.root || r.plist !== p.plist || typeof r.version !== 'string' || !/^\d+\.\d+\.\d+$/u.test(r.version) || typeof r.node !== 'string' || typeof r.cli !== 'string' || typeof r.launcher !== 'string' || typeof r.plistHash !== 'string' || typeof r.launcherHash !== 'string' || !Array.isArray(r.pins) || r.pins.length > 10000)
42
+ throw new StoreError('integrity');
43
+ if (r.launcher !== join(p.root, 'launchers', r.version + '-' + r.id, 'run.mjs') || !/^\/[\s\S]+/u.test(r.node) || !/^\/[\s\S]+/u.test(r.cli))
44
+ throw new StoreError('integrity');
45
+ for (const pin of r.pins) {
46
+ const f = exact(pin, ['path', 'bytes', 'sha256']);
47
+ if (typeof f.path !== 'string' || resolve(f.path) !== f.path || typeof f.bytes !== 'number' || !Number.isSafeInteger(f.bytes) || f.bytes < 0 || typeof f.sha256 !== 'string' || !/^[a-f0-9]{64}$/u.test(f.sha256))
48
+ throw new StoreError('integrity');
49
+ }
50
+ return r;
51
+ }
52
+ function verifyOwnedFiles(record) { for (const [path, sha] of [[record.plist, record.plistHash], [record.launcher, record.launcherHash]]) {
53
+ if (describe(path, path, 4 * 1024 * 1024).sha256 !== sha)
54
+ throw new StoreError('integrity');
55
+ } }
56
+ function current(record) { try {
57
+ return record.pins.every(p => { const r = describe(p.path, p.path, 512 * 1024 ** 2); return r.bytes === p.bytes && r.sha256 === p.sha256; });
58
+ }
59
+ catch {
60
+ return false;
61
+ } }
62
+ function xml(value) { return value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;').replaceAll("'", '&apos;'); }
63
+ export function agentPlist(record) { return `<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0"><dict><key>Label</key><string>${xml(record.label)}</string><key>ProgramArguments</key><array><string>${xml(record.node)}</string><string>${xml(record.launcher)}</string></array><key>RunAtLoad</key><true/><key>StartInterval</key><integer>30</integer><key>ProcessType</key><string>Background</string><key>ExitTimeOut</key><integer>60</integer><key>Umask</key><integer>63</integer><key>EnvironmentVariables</key><dict><key>PATH</key><string>/usr/bin:/bin</string><key>LANG</key><string>en_US.UTF-8</string><key>HRANESS_NO_UPDATE</key><string>1</string></dict><key>StandardOutPath</key><string>/dev/null</string><key>StandardErrorPath</key><string>/dev/null</string></dict></plist>\n`; }
64
+ function pinPackage(env) {
65
+ const cli = realpathSync(env.cli), root = dirname(dirname(cli));
66
+ if (cli !== join(root, 'bin', 'sponge.js'))
67
+ throw new StoreError('invalid-input');
68
+ const pkg = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'));
69
+ if (pkg.name !== '@hraness/sponge' || pkg.version !== env.version || !/^\d+\.\d+\.\d+$/u.test(env.version))
70
+ throw new StoreError('invalid-input');
71
+ const paths = [env.node, cli, join(root, 'package.json')];
72
+ let total = 0;
73
+ const visit = (directory) => { safeDirectory(directory); for (const name of readdirSync(directory).sort()) {
74
+ const path = join(directory, name), stat = lstatSync(path);
75
+ if (stat.isSymbolicLink())
76
+ throw new StoreError('integrity');
77
+ if (stat.isDirectory())
78
+ visit(path);
79
+ else {
80
+ regularFile(path);
81
+ if (paths.length >= 10000 || total + stat.size > 64 * 1024 * 1024)
82
+ throw new StoreError('limit-exceeded');
83
+ total += stat.size;
84
+ paths.push(path);
85
+ }
86
+ } };
87
+ visit(join(root, 'dist'));
88
+ const catalog = join(root, 'native', 'runtime-manifest.json');
89
+ let catalogPresent = false;
90
+ try {
91
+ lstatSync(catalog);
92
+ catalogPresent = true;
93
+ }
94
+ catch (error) {
95
+ if (error.code !== 'ENOENT')
96
+ throw error;
97
+ }
98
+ if (catalogPresent) {
99
+ safeDirectory(dirname(catalog));
100
+ regularFile(catalog);
101
+ if (lstatSync(catalog).size > 16384 || paths.length >= 10000)
102
+ throw new StoreError('limit-exceeded');
103
+ paths.push(catalog);
104
+ }
105
+ return paths.map(path => { const record = describe(path, path, 512 * 1024 ** 2); return { path, bytes: record.bytes, sha256: record.sha256 }; });
106
+ }
107
+ function launcher(record) { return `// Sponge pinned service launcher. Reinstall the worker after upgrading its package.\nimport {openSync,readSync,closeSync,fstatSync,constants} from 'node:fs';\nimport {createHash} from 'node:crypto';\nimport {pathToFileURL} from 'node:url';\nconst pins=${JSON.stringify(record.pins)};\nfor(const p of pins){let fd;try{fd=openSync(p.path,constants.O_RDONLY|constants.O_NOFOLLOW);const s=fstatSync(fd);if(!s.isFile()||s.nlink!==1||s.size!==p.bytes)throw Error();const h=createHash('sha256'),b=Buffer.alloc(65536);let n;while((n=readSync(fd,b,0,b.length,null)))h.update(b.subarray(0,n));if(h.digest('hex')!==p.sha256)throw Error();}catch{process.stderr.write('Sponge worker installation changed. Run sponge worker uninstall, then sponge worker install.\\n');process.exit(78);}finally{if(fd!==undefined)closeSync(fd);}}\nprocess.env.SPONGE_SERVICE_INSTALLATION=${JSON.stringify(record.id)};process.env.HRANESS_NO_UPDATE='1';process.argv=[process.execPath,${JSON.stringify(record.cli)},'--library',${JSON.stringify(record.library)},'worker','serve',...${JSON.stringify(record.language ? ['--language', record.language] : [])}];\nawait import(pathToFileURL(${JSON.stringify(record.cli)}).href);\n`; }
108
+ async function loaded(env, target) { const result = await env.launchctl(['print', target]); if (result.code === 0)
109
+ return true; if (result.code === 113)
110
+ return false; throw new StoreError('unavailable'); }
111
+ export async function serviceStatus(store, env) { const p = paths(store, env), record = installation(store, env); if (!record)
112
+ return { installed: false }; verifyOwnedFiles(record); const stale = !current(record); return { installed: true, loaded: await loaded(env, p.target), version: record.version, language: record.language, stale, ...(stale ? { repair: 'Run sponge worker uninstall, then sponge worker install from the current installed package.' } : {}) }; }
113
+ export async function installService(store, env, options = {}) {
114
+ if (options.language !== undefined && options.language !== 'en')
115
+ throw new StoreError('invalid-input');
116
+ const p = paths(store, env);
117
+ safeDirectory(p.root, true);
118
+ const previous = installation(store, env);
119
+ if (previous) {
120
+ if (previous.language !== (options.language ?? null))
121
+ throw new StoreError('invalid-input');
122
+ return serviceStatus(store, env);
123
+ }
124
+ if (await loaded(env, p.target))
125
+ throw new StoreError('owner-busy');
126
+ safeDirectory(p.agents, true);
127
+ if (existsSync(p.plist))
128
+ throw new StoreError('integrity');
129
+ const id = randomUUID(), pins = pinPackage(env), launcherPath = join(p.root, 'launchers', env.version + '-' + id, 'run.mjs');
130
+ safeDirectory(dirname(launcherPath), true);
131
+ const base = { language: options.language ?? null, schemaVersion: 1, id, label: p.label, uid: env.uid, version: env.version, node: env.node, cli: env.cli, library: store.root, plist: p.plist, launcher: launcherPath, pins };
132
+ const code = launcher(base), plist = agentPlist(base);
133
+ writePrivateExclusive(launcherPath, code);
134
+ writePrivateExclusive(p.plist, plist);
135
+ const record = { ...base, launcherHash: digest(code), plistHash: digest(plist) };
136
+ writePrivateExclusive(p.record, JSON.stringify(record) + '\n');
137
+ const result = await env.launchctl(['bootstrap', `gui/${env.uid}`, p.plist]);
138
+ if (result.code !== 0)
139
+ throw new StoreError('unavailable');
140
+ return serviceStatus(store, env);
141
+ }
142
+ export async function uninstallService(store, env) {
143
+ const p = paths(store, env), record = installation(store, env);
144
+ if (!record)
145
+ return { installed: false };
146
+ verifyOwnedFiles(record);
147
+ if (await loaded(env, p.target)) {
148
+ const result = await env.launchctl(['bootout', p.target]);
149
+ if (result.code !== 0 && result.code !== 113)
150
+ throw new StoreError('unavailable');
151
+ }
152
+ for (let attempt = 0; attempt < 50; attempt++) {
153
+ let alive = false;
154
+ const run = join(p.root, 'run.json');
155
+ if (existsSync(run)) {
156
+ const r = exact(readJson(run), ['schemaVersion', 'installationId', 'runId', 'identity', 'state']), identity = exact(r.identity, ['pid', 'uid', 'started']);
157
+ if (r.schemaVersion !== 1 || r.installationId !== record.id || typeof r.runId !== 'string' || !/^[a-f0-9-]{36}$/u.test(r.runId) || !['running', 'stopped', 'cleanup-required'].includes(String(r.state)) || !Number.isSafeInteger(identity.pid) || Number(identity.pid) <= 1 || identity.uid !== env.uid || typeof identity.started !== 'string' || !identity.started || identity.started.length > 256)
158
+ throw new StoreError('integrity');
159
+ alive = await env.alive(identity);
160
+ }
161
+ if (!await loaded(env, p.target) && !alive && store.ownerSnapshot() === null && store.pendingCleanupCount() === 0 && await recoverServiceLifetime(store, record.id, env.alive)) {
162
+ verifyOwnedFiles(record);
163
+ unlinkSync(record.plist);
164
+ unlinkSync(record.launcher);
165
+ if (existsSync(run))
166
+ unlinkSync(run);
167
+ unlinkSync(p.record);
168
+ syncDirectory(p.agents);
169
+ syncDirectory(dirname(record.launcher));
170
+ syncDirectory(p.root);
171
+ return { installed: false, cleanup: 'settled' };
172
+ }
173
+ await env.pause(100);
174
+ }
175
+ throw Object.assign(new StoreError('owner-busy'), { reason: 'worker-cleanup-required' });
176
+ }
177
+ export function installedServiceId(store, env) { const record = installation(store, env); if (!record || process.env.SPONGE_SERVICE_INSTALLATION !== record.id)
178
+ throw new StoreError('invalid-input'); verifyOwnedFiles(record); if (!current(record))
179
+ throw new StoreError('unavailable'); return record.id; }
@@ -0,0 +1,7 @@
1
+ import type { ProcessIdentity } from '../worker/identity.js';
2
+ import { type LibraryStore } from '../store/index.js';
3
+ export declare function acquireServiceLifetime(root: string, installationId: string, identity: ProcessIdentity): Promise<{
4
+ release: () => Promise<void>;
5
+ }>;
6
+ /** Called only after bootout or a failed prior run; never signal a recorded PID. */
7
+ export declare function recoverServiceLifetime(store: LibraryStore, installationId: string, alive: (identity: ProcessIdentity) => Promise<boolean>): Promise<boolean>;
@@ -0,0 +1,73 @@
1
+ import { join } from 'node:path';
2
+ import { lstat, link, unlink } from 'node:fs/promises';
3
+ import { randomUUID } from 'node:crypto';
4
+ import { readJournal, writeJournal, syncDir } from '../worker/journal.js';
5
+ import { StoreError } from '../store/index.js';
6
+ const pending = () => new StoreError('owner-busy');
7
+ function id(value) { return typeof value === 'string' && /^[a-f0-9-]{36}$/u.test(value); }
8
+ async function same(a, b) { const [x, y] = await Promise.all([lstat(a), lstat(b)]); if (!x.isFile() || x.isSymbolicLink() || !y.isFile() || y.isSymbolicLink() || x.dev !== y.dev || x.ino !== y.ino)
9
+ throw pending(); }
10
+ async function read(path) { const r = await readJournal(path); if (!r || Object.keys(r).sort().join(',') !== 'id,identity,installationId,schemaVersion' || r.schemaVersion !== 1 || !id(r.id) || !id(r.installationId) || !r.identity || Object.keys(r.identity).sort().join(',') !== 'pid,started,uid' || !Number.isSafeInteger(r.identity.pid) || r.identity.pid <= 1 || !Number.isSafeInteger(r.identity.uid) || r.identity.uid < 0 || typeof r.identity.started !== 'string' || !r.identity.started || r.identity.started.length > 256)
11
+ throw pending(); return r; }
12
+ export async function acquireServiceLifetime(root, installationId, identity) {
13
+ const value = { schemaVersion: 1, id: randomUUID(), installationId, identity }, file = join(root, `lifetime-${value.id}.json`), lock = join(root, 'lifetime.lock');
14
+ await writeJournal(file, value);
15
+ try {
16
+ await link(file, lock);
17
+ await syncDir(root);
18
+ }
19
+ catch (error) {
20
+ await unlink(file);
21
+ if (error.code === 'EEXIST')
22
+ throw pending();
23
+ throw error;
24
+ }
25
+ return { release: async () => { await same(file, lock); await unlink(lock); await unlink(file); await syncDir(root); } };
26
+ }
27
+ /** Called only after bootout or a failed prior run; never signal a recorded PID. */
28
+ export async function recoverServiceLifetime(store, installationId, alive) {
29
+ const root = join(store.root, 'runtime', 'service'), lock = join(root, 'lifetime.lock');
30
+ let value;
31
+ try {
32
+ value = await read(lock);
33
+ }
34
+ catch (error) {
35
+ if (error.code === 'ENOENT')
36
+ return true;
37
+ throw error;
38
+ }
39
+ if (value.installationId !== installationId || await alive(value.identity))
40
+ return false;
41
+ const file = join(root, `lifetime-${value.id}.json`), guard = join(root, 'lifetime-recovery.lock');
42
+ await same(file, lock);
43
+ try {
44
+ await link(file, guard);
45
+ }
46
+ catch (error) {
47
+ if (error.code === 'EEXIST')
48
+ return false;
49
+ throw error;
50
+ }
51
+ try {
52
+ await same(file, lock);
53
+ if (await alive(value.identity) || store.ownerSnapshot() !== null || store.pendingCleanupCount() > 0)
54
+ return false;
55
+ await unlink(lock);
56
+ await syncDir(root);
57
+ return true;
58
+ }
59
+ finally {
60
+ await same(file, guard);
61
+ await unlink(guard);
62
+ try {
63
+ await lstat(lock);
64
+ }
65
+ catch (error) {
66
+ if (error.code === 'ENOENT')
67
+ await unlink(file);
68
+ else
69
+ throw error;
70
+ }
71
+ await syncDir(root);
72
+ }
73
+ }
@@ -0,0 +1,21 @@
1
+ import type { LibraryStore } from '../store/index.js';
2
+ import { type ProcessIdentity } from '../worker/identity.js';
3
+ export type ServiceEvent = 'started' | 'idle' | 'drained' | 'busy' | 'repair-required' | 'stopped';
4
+ /** Fixed event vocabulary; source URLs, credentials, content and raw errors never enter logs. */
5
+ export declare function serviceLog(root: string, event: ServiceEvent): void;
6
+ export declare function idleWait(ms: number, signal: AbortSignal): Promise<void>;
7
+ export interface ServiceLoopOptions {
8
+ signal: AbortSignal;
9
+ installationId: string;
10
+ drain: (signal: AbortSignal) => Promise<readonly unknown[]>;
11
+ idleMs?: number;
12
+ wait?: (ms: number, signal: AbortSignal) => Promise<void>;
13
+ log?: (event: ServiceEvent) => void;
14
+ onIdle?: () => Promise<void>;
15
+ identity?: () => Promise<ProcessIdentity | null>;
16
+ }
17
+ /** Uses the existing drain engine; it owns locking, cancellation and descendant collection. */
18
+ export declare function runWorkerService(store: LibraryStore, options: ServiceLoopOptions): Promise<{
19
+ cycles: number;
20
+ cleanup: 'settled' | 'pending';
21
+ }>;