@hraness/sponge 0.2.6 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +21 -9
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +38 -16
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +9 -46
  64. package/dist/src/digest/editorial.js +57 -49
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +49 -28
  67. package/dist/src/digest/pipeline.js +343 -100
  68. package/dist/src/digest/schema.d.ts +11 -25
  69. package/dist/src/digest/schema.js +86 -18
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +3 -6
  82. package/dist/src/importers/jungle-digest.js +196 -69
  83. package/dist/src/importers/jungle-store.d.ts +15 -6
  84. package/dist/src/importers/jungle-store.js +122 -28
  85. package/dist/src/importers/jungle.d.ts +14 -14
  86. package/dist/src/importers/jungle.js +136 -31
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +106 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -79
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -3,9 +3,9 @@ import { inspectNative } from '../models/runtime.js';
3
3
  import { drainIndexQueue } from '../indexing/runner.js';
4
4
  import { drainLibrary } from '../worker/basic.js';
5
5
  import { recoverServiceLifetime } from '../service/lifetime.js';
6
- import { StoreError } from '../store/types.js';
7
6
  import { runWorkerService } from '../service/loop.js';
8
- import { installService, uninstallService, serviceStatus, installedServiceId, serviceEnvironment } from '../service/launch-agent.js';
7
+ import { installService, uninstallService, serviceStatus, installedServiceId, serviceEnvironment, } from '../service/launch-agent.js';
8
+ import { SpongeError, codeOf } from '../errors.js';
9
9
  export async function workerAction(store, action, version, environment, options = {}) {
10
10
  const env = environment ?? serviceEnvironment(version);
11
11
  if (action === 'install')
@@ -14,39 +14,74 @@ export async function workerAction(store, action, version, environment, options
14
14
  return uninstallService(store, env);
15
15
  return serviceStatus(store, env);
16
16
  }
17
+ /** Doctor's read-only view of the background worker: never installs, repairs or signals anything. */
18
+ export async function workerDoctor(store, version, environment) {
19
+ let env;
20
+ try {
21
+ env = environment ?? serviceEnvironment(version);
22
+ }
23
+ catch {
24
+ return { worker: 'unavailable' };
25
+ }
26
+ if (env.platform !== 'darwin')
27
+ return { worker: 'unsupported' };
28
+ try {
29
+ const s = await serviceStatus(store, env);
30
+ if (!s.installed)
31
+ return { worker: 'not-installed' };
32
+ const exit = s.lastExit === undefined ? {} : { lastExit: s.lastExit };
33
+ return s.stale
34
+ ? { worker: 'drift', drift: s.drift, repair: s.repair, ...exit }
35
+ : {
36
+ worker: s.loaded ? 'ready' : 'not-loaded',
37
+ ...exit,
38
+ ...(s.loaded ? {} : { repair: 'Run sponge worker install to load it again.' }),
39
+ };
40
+ }
41
+ catch (error) {
42
+ return { worker: codeOf(error, 'unavailable') };
43
+ }
44
+ }
17
45
  /** Dispatch only work that claimNext can admit; delayed retries retain their due time. */
18
46
  export function hasDueAcquisition(store) {
19
47
  const now = new Date().toISOString();
20
- return store.listJobs().some(job => job.operation !== 'index' && ['queued', 'delayed'].includes(String(job.state)) && job.cancel_requested !== 1 && (job.not_before === null || job.not_before === undefined || String(job.not_before) <= now));
48
+ return store.queue
49
+ .list()
50
+ .some((job) => job.operation !== 'index' &&
51
+ ['queued', 'delayed'].includes(String(job.state)) &&
52
+ job.cancel_requested !== 1 &&
53
+ (job.not_before === null || job.not_before === undefined || String(job.not_before) <= now));
21
54
  }
22
55
  export async function drainServiceQueues(store, options, version, signal) {
23
- if (store.pendingCleanupCount())
24
- throw Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
56
+ if (store.queue.pendingCleanup())
57
+ throw new SpongeError('cleanup-required');
25
58
  const results = [];
26
59
  let acquisitionError;
27
60
  if (hasDueAcquisition(store)) {
28
61
  try {
29
- const runtime = options.runtime ?? await options.resolveCaptureRuntime?.();
62
+ const runtime = options.runtime ?? (await options.resolveCaptureRuntime?.());
30
63
  if (!runtime)
31
- throw Object.assign(Error('runtime-missing'), { code: 'runtime-missing' });
64
+ throw new SpongeError('runtime-missing');
32
65
  let asrRuntime = options.asrRuntime;
33
66
  try {
34
67
  asrRuntime ??= await inspectNative('asr', join(store.root, 'runtime', 'models'), version);
35
68
  }
36
69
  catch (error) {
37
- if (!['runtime-missing', 'unsupported'].includes(String(error.code)))
70
+ if (!['runtime-missing', 'unsupported'].includes(codeOf(error)))
38
71
  throw error;
39
72
  }
40
- results.push(...await drainLibrary(store, { ...options, runtime, asrRuntime, signal, allowPrivateNetwork: false }));
73
+ results.push(...(await drainLibrary(store, { ...options, runtime, asrRuntime, signal, allowPrivateNetwork: false })));
41
74
  }
42
75
  catch (error) {
43
76
  acquisitionError = error;
44
77
  }
45
78
  }
46
79
  // A missing capture setup must not starve already queued, independently runnable indexes.
47
- if (!signal.aborted && store.pendingCleanupCount() === 0 && store.listJobs().some(job => job.operation === 'index' && job.state === 'queued')) {
80
+ if (!signal.aborted &&
81
+ store.queue.pendingCleanup() === 0 &&
82
+ store.queue.list().some((job) => job.operation === 'index' && job.state === 'queued')) {
48
83
  const runtime = await inspectNative('embedding', join(store.root, 'runtime', 'models'), version);
49
- results.push(...await drainIndexQueue(store, { runtime, signal }));
84
+ results.push(...(await drainIndexQueue(store, { runtime, signal })));
50
85
  }
51
86
  if (acquisitionError)
52
87
  throw acquisitionError;
@@ -55,8 +90,12 @@ export async function drainServiceQueues(store, options, version, signal) {
55
90
  export async function serveWorker(store, options, version, environment) {
56
91
  const env = environment ?? serviceEnvironment(version), installationId = installedServiceId(store, env);
57
92
  if (!options.signal)
58
- throw Error('signal-required');
59
- if (!await recoverServiceLifetime(store, installationId, env.alive))
60
- throw new StoreError('owner-busy');
61
- return runWorkerService(store, { installationId, signal: options.signal, drain: signal => drainServiceQueues(store, options, version, signal) });
93
+ throw new SpongeError('invalid-input');
94
+ if (!(await recoverServiceLifetime(store, installationId, env.alive)))
95
+ throw new SpongeError('owner-busy');
96
+ return runWorkerService(store, {
97
+ installationId,
98
+ signal: options.signal,
99
+ drain: (signal) => drainServiceQueues(store, options, version, signal),
100
+ });
62
101
  }
@@ -0,0 +1,34 @@
1
+ import { type Derivation, type DigestCandidate, type DigestReview } from '../digest/derivation.js';
2
+ import { type Editorial } from '../digest/editorial.js';
3
+ import type { JungleEditorial } from '../importers/jungle-digest.js';
4
+ import type { DerivationKind, ReviewStatus } from './types.js';
5
+ /** The derivation-kind registry: the one place a kind name maps to its rules. Add a kind here and nowhere else. */
6
+ export declare const derivationKinds: {
7
+ readonly digest: DerivationKind<"digest", import("../digest/schema.js").Digest, DigestReview, DigestReview, DigestCandidate>;
8
+ readonly editorial: DerivationKind<"editorial", JungleEditorial, {
9
+ note?: ReviewStatus;
10
+ }, ReviewStatus | null, {
11
+ output: JungleEditorial;
12
+ }>;
13
+ };
14
+ export type Kind = keyof typeof derivationKinds;
15
+ /** Per kind: the stored derivation, what a review takes, and what storing takes besides item, revision and producer. */
16
+ export interface KindTypes {
17
+ digest: {
18
+ derivation: Derivation;
19
+ review: DigestReview;
20
+ candidate: DigestCandidate;
21
+ };
22
+ editorial: {
23
+ derivation: Editorial;
24
+ review: ReviewStatus | null;
25
+ candidate: {
26
+ output: JungleEditorial;
27
+ };
28
+ };
29
+ }
30
+ export type DerivationOf<K extends Kind> = KindTypes[K]['derivation'];
31
+ /** Publishing precedence: an item publishes from the first kind that has a current derivation, and never from two. */
32
+ export declare const publishOrder: readonly Kind[];
33
+ /** Dynamic dispatch on a stored kind name; an unknown kind is invalid input (it may arrive through the change log). */
34
+ export declare function kindOf(name: unknown): DerivationKind<Kind, any, any, any, any>;
@@ -0,0 +1,14 @@
1
+ import { digestKind } from '../digest/derivation.js';
2
+ import { editorialKind } from '../digest/editorial.js';
3
+ import { SpongeError } from '../errors.js';
4
+ /** The derivation-kind registry: the one place a kind name maps to its rules. Add a kind here and nowhere else. */
5
+ export const derivationKinds = { digest: digestKind, editorial: editorialKind };
6
+ /** Publishing precedence: an item publishes from the first kind that has a current derivation, and never from two. */
7
+ export const publishOrder = ['digest', 'editorial'];
8
+ const isKind = (value) => typeof value === 'string' && Object.hasOwn(derivationKinds, value);
9
+ /** Dynamic dispatch on a stored kind name; an unknown kind is invalid input (it may arrive through the change log). */
10
+ export function kindOf(name) {
11
+ if (!isKind(name))
12
+ throw new SpongeError('invalid-input');
13
+ return derivationKinds[name];
14
+ }
@@ -0,0 +1,48 @@
1
+ /** A review decision on one field of a derivation. Only accepted fields can ever be published. */
2
+ export type ReviewStatus = 'accepted' | 'rejected';
3
+ /** The public digest shape: what `sponge publish` may carry for an item. */
4
+ export interface PublicDigest {
5
+ lead?: string;
6
+ ideas: {
7
+ claim: string;
8
+ detail: string;
9
+ kind?: string;
10
+ quotes: string[];
11
+ }[];
12
+ }
13
+ /** One stored derivation: immutable output addressed by content, plus its review (user state, written only through the change log). */
14
+ export interface StoredDerivation<K extends string, Output, Review> {
15
+ id: string;
16
+ itemId: string;
17
+ revisionId: string;
18
+ kind: K;
19
+ producer: unknown;
20
+ inputHash: string;
21
+ output: Output;
22
+ review: Review;
23
+ createdAt: string;
24
+ }
25
+ /** What a kind admits for storage: canonical output JSON, the input hash it is addressed under, and the stored schema version. */
26
+ interface Admitted {
27
+ output: string;
28
+ inputHash: string;
29
+ schemaVersion: number;
30
+ }
31
+ /**
32
+ * One derivation kind. The store owns the SQL; a kind owns its rules. Every place that stores, reviews, replays or publishes a
33
+ * derivation consults the kind registry (`kinds.ts`), never a kind name.
34
+ */
35
+ export interface DerivationKind<K extends string, Output, Review, ReviewInput, Candidate> {
36
+ readonly kind: K;
37
+ /** The schema version new objects of this kind are written at. */
38
+ readonly schemaVersion: number;
39
+ /** Validate a candidate against the Markdown of the revision it derives from (read lazily), or throw. */
40
+ admit(candidate: Candidate, source: () => string): Admitted;
41
+ /** The review paths a review input sets, with their new values, validated against the stored derivation. */
42
+ reviewPaths(d: StoredDerivation<K, Output, Review>, input: ReviewInput): [path: string, value: ReviewStatus | null][];
43
+ /** Replay one logged review path onto the stored review: the change-log reducer, so it validates untrusted input. */
44
+ applyReview(review: Review, output: Output, path: string, value: ReviewStatus | null): Review;
45
+ /** The publishable subset, for kinds that publish. */
46
+ publish?(d: StoredDerivation<K, Output, Review>, source: () => string): PublicDigest | undefined;
47
+ }
48
+ export {};
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,25 @@
1
+ import { type Digest, type DigestOptions } from './schema.js';
2
+ import type { DerivationKind, PublicDigest, ReviewStatus, StoredDerivation } from '../derivations/types.js';
3
+ /** Per-field review of a digest. Only accepted fields can ever be published. */
4
+ export interface DigestReview {
5
+ gist?: ReviewStatus;
6
+ ideas?: (ReviewStatus | null)[];
7
+ quotes?: (ReviewStatus | null)[][];
8
+ }
9
+ export type Derivation = StoredDerivation<'digest', Digest, DigestReview>;
10
+ /** What a digest is stored from: the producer's input hash, the model output, and the checker options it was drafted under. */
11
+ export interface DigestCandidate {
12
+ inputHash: string;
13
+ output: unknown;
14
+ options?: DigestOptions;
15
+ }
16
+ /** A parsed digest serialized in fixed key order, so key order never makes a different object. Optional fields appear only when present. */
17
+ export declare function canonicalDigest(d: Digest): string;
18
+ /** The accepted subset of a digest (v1 or v2). Rejected or unreviewed fields never leave the device; v1 ideas keep their kind. */
19
+ export declare function acceptedDigest(d: Derivation): PublicDigest | undefined;
20
+ /**
21
+ * Digest: a draft is stored only when it is Digest v2 and every v2 rule passes against the revision it was made from.
22
+ * Stored v1 digests stay readable; new ones are v2. Each reviewed field is its own logged path: `review.gist`,
23
+ * `review.ideas.<i>`, `review.quotes.<i>.<j>`.
24
+ */
25
+ export declare const digestKind: DerivationKind<'digest', Digest, DigestReview, DigestReview, DigestCandidate>;
@@ -0,0 +1,124 @@
1
+ import { checkDigest, digestSchemaVersion, parseDigest } from './schema.js';
2
+ import { hash } from '../store/files.js';
3
+ import { SpongeError } from '../errors.js';
4
+ const fail = () => {
5
+ throw new SpongeError('invalid-input');
6
+ };
7
+ /** A parsed digest serialized in fixed key order, so key order never makes a different object. Optional fields appear only when present. */
8
+ export function canonicalDigest(d) {
9
+ return JSON.stringify({
10
+ schemaVersion: d.schemaVersion,
11
+ form: d.form,
12
+ gist: { lead: d.gist.lead, ...(d.gist.context !== undefined ? { context: d.gist.context } : {}) },
13
+ ideas: d.ideas.map((i) => ({
14
+ claim: i.claim,
15
+ detail: i.detail,
16
+ ...(i.kind !== undefined ? { kind: i.kind } : {}),
17
+ ...(i.voice
18
+ ? { voice: i.voice.type === 'author' ? { type: 'author' } : { type: i.voice.type, name: i.voice.name } }
19
+ : {}),
20
+ quotes: i.quotes.map((q) => ({
21
+ text: q.text,
22
+ ...(q.speaker !== undefined ? { speaker: q.speaker } : {}),
23
+ ...(q.attribution !== undefined ? { attribution: q.attribution } : {}),
24
+ ...(q.locator
25
+ ? {
26
+ locator: q.locator.type === 'timestamp'
27
+ ? { type: 'timestamp', ms: q.locator.ms }
28
+ : { type: 'page', page: q.locator.page },
29
+ }
30
+ : {}),
31
+ })),
32
+ })),
33
+ tags: [...d.tags],
34
+ });
35
+ }
36
+ /** The accepted subset of a digest (v1 or v2). Rejected or unreviewed fields never leave the device; v1 ideas keep their kind. */
37
+ export function acceptedDigest(d) {
38
+ const ideas = d.output.ideas.flatMap((idea, i) => d.review.ideas?.[i] === 'accepted'
39
+ ? [
40
+ {
41
+ claim: idea.claim,
42
+ detail: idea.detail,
43
+ ...(idea.kind !== undefined ? { kind: idea.kind } : {}),
44
+ quotes: idea.quotes.filter((_, j) => d.review.quotes?.[i]?.[j] === 'accepted').map((q) => q.text),
45
+ },
46
+ ]
47
+ : []);
48
+ const lead = d.review.gist === 'accepted' ? d.output.gist.lead : undefined;
49
+ return lead || ideas.length ? { ...(lead ? { lead } : {}), ideas } : undefined;
50
+ }
51
+ const status = (v) => v === null || v === 'accepted' || v === 'rejected';
52
+ /**
53
+ * Digest: a draft is stored only when it is Digest v2 and every v2 rule passes against the revision it was made from.
54
+ * Stored v1 digests stay readable; new ones are v2. Each reviewed field is its own logged path: `review.gist`,
55
+ * `review.ideas.<i>`, `review.quotes.<i>.<j>`.
56
+ */
57
+ export const digestKind = {
58
+ kind: 'digest',
59
+ schemaVersion: digestSchemaVersion,
60
+ admit(candidate, source) {
61
+ hash(candidate.inputHash);
62
+ const parsed = parseDigest(candidate.output);
63
+ if (!parsed.digest)
64
+ throw new SpongeError('invalid-input', { issues: parsed.issues });
65
+ if (parsed.digest.schemaVersion !== digestSchemaVersion)
66
+ throw new SpongeError('invalid-input', { issues: [{ path: 'schemaVersion', code: 'superseded' }] });
67
+ const check = checkDigest(parsed.digest, source(), candidate.options);
68
+ if (!check.ok)
69
+ throw new SpongeError('invalid-input', { issues: check.issues });
70
+ const output = canonicalDigest(parsed.digest);
71
+ if (Buffer.byteLength(output) > 65536)
72
+ fail();
73
+ return { output, inputHash: candidate.inputHash, schemaVersion: parsed.digest.schemaVersion };
74
+ },
75
+ reviewPaths(d, review) {
76
+ if (typeof review !== 'object' ||
77
+ review === null ||
78
+ Object.keys(review).some((k) => !['gist', 'ideas', 'quotes'].includes(k)))
79
+ fail();
80
+ if (review.gist !== undefined && review.gist !== 'accepted' && review.gist !== 'rejected')
81
+ fail();
82
+ if (review.ideas !== undefined &&
83
+ (!Array.isArray(review.ideas) || review.ideas.length !== d.output.ideas.length || !review.ideas.every(status)))
84
+ fail();
85
+ if (review.quotes !== undefined &&
86
+ (!Array.isArray(review.quotes) ||
87
+ review.quotes.length !== d.output.ideas.length ||
88
+ review.quotes.some((q, i) => !Array.isArray(q) || q.length !== d.output.ideas[i].quotes.length || !q.every(status))))
89
+ fail();
90
+ const paths = [];
91
+ if (review.gist !== undefined)
92
+ paths.push(['review.gist', review.gist, d.review.gist ?? null]);
93
+ review.ideas?.forEach((v, i) => paths.push([`review.ideas.${i}`, v, d.review.ideas?.[i] ?? null]));
94
+ review.quotes?.forEach((q, i) => q.forEach((v, j) => paths.push([`review.quotes.${i}.${j}`, v, d.review.quotes?.[i]?.[j] ?? null])));
95
+ return paths.filter(([, next, prior]) => next !== prior).map(([path, next]) => [path, next]);
96
+ },
97
+ /** Review JSON in one canonical shape, whatever order its paths were written in. */
98
+ applyReview(review, output, path, value) {
99
+ const ideas = output.ideas.map((_, i) => review.ideas?.[i] ?? null);
100
+ const quotes = output.ideas.map((idea, i) => idea.quotes.map((_, j) => review.quotes?.[i]?.[j] ?? null));
101
+ let gist = review.gist ?? null;
102
+ const m = /^review\.(?:(gist)|ideas\.(\d{1,2})|quotes\.(\d{1,2})\.(\d{1,2}))$/.exec(path) ?? fail();
103
+ if (m[1])
104
+ gist = value;
105
+ else if (m[2] !== undefined) {
106
+ const i = Number(m[2]);
107
+ if (i >= ideas.length)
108
+ fail();
109
+ ideas[i] = value;
110
+ }
111
+ else {
112
+ const i = Number(m[3]), j = Number(m[4]);
113
+ if (i >= quotes.length || j >= quotes[i].length)
114
+ fail();
115
+ quotes[i][j] = value;
116
+ }
117
+ return {
118
+ ...(gist ? { gist } : {}),
119
+ ...(ideas.some((v) => v) ? { ideas } : {}),
120
+ ...(quotes.some((q) => q.some((v) => v)) ? { quotes } : {}),
121
+ };
122
+ },
123
+ publish: (d) => acceptedDigest(d),
124
+ };
@@ -1,54 +1,17 @@
1
- import type { DatabaseSync } from 'node:sqlite';
2
1
  import { type JungleEditorial } from '../importers/jungle-digest.js';
3
- import { type ReviewStatus } from '../store/changes.js';
2
+ import type { DerivationKind, ReviewStatus, StoredDerivation } from '../derivations/types.js';
4
3
  /**
5
4
  * An editorial derivation keeps a reviewed source note whole (today: one Jungle Reading note of any form), without
6
5
  * Digest's limits. It is immutable data addressed like a digest; its review is one logged path, `review.note`.
7
6
  */
8
- export interface Editorial {
9
- id: string;
10
- itemId: string;
11
- revisionId: string;
12
- kind: 'editorial';
13
- producer: unknown;
14
- inputHash: string;
15
- output: JungleEditorial;
16
- review: {
17
- note?: ReviewStatus;
18
- };
19
- createdAt: string;
20
- }
21
- /** Bound on one editorial object: Jungle's largest notes (full transcripts) are a few hundred KiB. */
22
- export declare const editorialBytes: number;
7
+ export type Editorial = StoredDerivation<'editorial', JungleEditorial, {
8
+ note?: ReviewStatus;
9
+ }>;
23
10
  /** Fixed key order, so the same note is the same object on every device. */
24
11
  export declare function canonicalEditorial(e: JungleEditorial): string;
25
- /** The input hash is the note's own Markdown hash: an edited note is a new object, an unchanged one a duplicate. */
26
- export declare function putEditorial(db: DatabaseSync, input: {
27
- itemId: string;
28
- revisionId: string;
29
- producer: Record<string, unknown>;
12
+ /** Editorial: the input hash is the note's own Markdown hash, so an edited note is a new object and an unchanged one a duplicate. */
13
+ export declare const editorialKind: DerivationKind<'editorial', JungleEditorial, {
14
+ note?: ReviewStatus;
15
+ }, ReviewStatus | null, {
30
16
  output: JungleEditorial;
31
- }): {
32
- id: string;
33
- duplicate: boolean;
34
- };
35
- export declare function readEditorial(db: DatabaseSync, id: string): Editorial | undefined;
36
- /** The newest editorial derivation of the item's current revision. */
37
- export declare function currentEditorial(db: DatabaseSync, itemId: string): Editorial | undefined;
38
- export declare function listEditorials(db: DatabaseSync, itemId: string): Editorial[];
39
- /** One logged change, and only when the status differs. */
40
- export declare function reviewEditorial(db: DatabaseSync, id: string, status: ReviewStatus | null): Editorial;
41
- /**
42
- * The public subset of an accepted editorial note, in the public digest shape and within Digest v2's limits: a lead only
43
- * when the note's first sentence fits the lead limit (never cut), claims with detail, and quotes that occur verbatim in the
44
- * capture with 4-25 words, assigned to ideas as the importer assigns them (at most 2 per idea and 6 in all), within the
45
- * per-entry budget (150 words, and the 10% source share with its 25-word floor).
46
- */
47
- export declare function publicEditorial(db: DatabaseSync, e: Editorial): {
48
- lead?: string;
49
- ideas: {
50
- claim: string;
51
- detail: string;
52
- quotes: string[];
53
- }[];
54
- } | undefined;
17
+ }>;
@@ -1,54 +1,34 @@
1
1
  import { locateAll, words } from '../annotations/quote.js';
2
- import { markdownFor } from '../annotations/store.js';
3
2
  import { assignQuotes, firstSentence } from '../importers/jungle-digest.js';
4
- import { mutate } from '../store/changes.js';
5
- import { StoreError } from '../store/types.js';
6
- import { digest } from '../store/files.js';
7
- import { derivationId } from './store.js';
3
+ import { sha256 } from '../util/hash.js';
8
4
  import { digestLimits as L } from './schema.js';
9
- const now = () => new Date().toISOString();
10
- const fail = (code = 'invalid-input') => { throw new StoreError(code); };
5
+ import { SpongeError } from '../errors.js';
6
+ const fail = () => {
7
+ throw new SpongeError('invalid-input');
8
+ };
11
9
  /** Bound on one editorial object: Jungle's largest notes (full transcripts) are a few hundred KiB. */
12
- export const editorialBytes = 1024 ** 2;
10
+ const editorialBytes = 1024 ** 2;
13
11
  const str = (v, max = 65536) => typeof v === 'string' && Buffer.byteLength(v) <= max;
14
12
  /** Fixed key order, so the same note is the same object on every device. */
15
13
  export function canonicalEditorial(e) {
16
- return JSON.stringify({ schemaVersion: e.schemaVersion, source: e.source, form: e.form, slug: e.slug, status: e.status, title: e.title, sourceUrl: e.sourceUrl, capture: e.capture, frontmatter: e.frontmatter, ...(e.gist !== undefined ? { gist: e.gist } : {}), ...(e.ideas ? { ideas: e.ideas.map(i => ({ claim: i.claim, detail: i.detail })) } : {}), ...(e.quotes ? { quotes: e.quotes.map(q => ({ text: q.text, attribution: q.attribution, speaker: q.speaker })) } : {}), markdownSha256: e.markdownSha256, markdown: e.markdown });
17
- }
18
- /** The input hash is the note's own Markdown hash: an edited note is a new object, an unchanged one a duplicate. */
19
- export function putEditorial(db, input) {
20
- const e = input.output, producer = JSON.stringify(input.producer);
21
- if (!producer || Buffer.byteLength(producer) > 4096)
22
- fail();
23
- if (db.prepare('SELECT item_id FROM revisions WHERE id=?').get(input.revisionId)?.item_id !== input.itemId)
24
- fail('unavailable');
25
- if (e?.schemaVersion !== 1 || e.source !== 'jungle-reading' || ![e.form, e.slug, e.status, e.title, e.sourceUrl, e.capture].every(v => str(v, 16384)) || !str(e.markdown, 4 * 1024 ** 2) || digest(e.markdown) !== e.markdownSha256)
26
- fail();
27
- const output = canonicalEditorial(e);
28
- if (Buffer.byteLength(output) > editorialBytes)
29
- fail();
30
- const id = derivationId(input.revisionId, 'editorial', e.markdownSha256, output);
31
- if (db.prepare('SELECT 1 FROM derivations WHERE id=?').get(id))
32
- return { id, duplicate: true };
33
- db.prepare("INSERT INTO derivations(id,item_id,revision_id,kind,schema_version,producer_json,input_hash,output_json,created_at) VALUES(?,?,?,'editorial',1,?,?,?,?)").run(id, input.itemId, input.revisionId, producer, e.markdownSha256, output, now());
34
- return { id, duplicate: false };
35
- }
36
- function editorialRow(r) { return { id: String(r.id), itemId: String(r.item_id), revisionId: String(r.revision_id), kind: 'editorial', producer: JSON.parse(String(r.producer_json)), inputHash: String(r.input_hash), output: JSON.parse(String(r.output_json)), review: JSON.parse(String(r.review_json)), createdAt: String(r.created_at) }; }
37
- export function readEditorial(db, id) { const r = db.prepare("SELECT * FROM derivations WHERE id=? AND kind='editorial'").get(id); return r ? editorialRow(r) : undefined; }
38
- /** The newest editorial derivation of the item's current revision. */
39
- export function currentEditorial(db, itemId) {
40
- const r = db.prepare("SELECT d.* FROM derivations d JOIN items i ON i.id=d.item_id AND i.current_revision=d.revision_id WHERE d.item_id=? AND d.kind='editorial' ORDER BY d.created_at DESC,d.rowid DESC LIMIT 1").get(itemId);
41
- return r ? editorialRow(r) : undefined;
42
- }
43
- export function listEditorials(db, itemId) { return db.prepare("SELECT * FROM derivations WHERE item_id=? AND kind='editorial' ORDER BY created_at,rowid").all(itemId).map(editorialRow); }
44
- /** One logged change, and only when the status differs. */
45
- export function reviewEditorial(db, id, status) {
46
- const d = readEditorial(db, id) ?? fail('unavailable');
47
- if (status !== null && status !== 'accepted' && status !== 'rejected')
48
- fail();
49
- if ((d.review.note ?? null) !== status)
50
- mutate(db, 'derivation', id, 'review.note', status);
51
- return readEditorial(db, id);
14
+ return JSON.stringify({
15
+ schemaVersion: e.schemaVersion,
16
+ source: e.source,
17
+ form: e.form,
18
+ slug: e.slug,
19
+ status: e.status,
20
+ title: e.title,
21
+ sourceUrl: e.sourceUrl,
22
+ capture: e.capture,
23
+ frontmatter: e.frontmatter,
24
+ ...(e.gist !== undefined ? { gist: e.gist } : {}),
25
+ ...(e.ideas ? { ideas: e.ideas.map((i) => ({ claim: i.claim, detail: i.detail })) } : {}),
26
+ ...(e.quotes
27
+ ? { quotes: e.quotes.map((q) => ({ text: q.text, attribution: q.attribution, speaker: q.speaker })) }
28
+ : {}),
29
+ markdownSha256: e.markdownSha256,
30
+ markdown: e.markdown,
31
+ });
52
32
  }
53
33
  /**
54
34
  * The public subset of an accepted editorial note, in the public digest shape and within Digest v2's limits: a lead only
@@ -56,15 +36,15 @@ export function reviewEditorial(db, id, status) {
56
36
  * capture with 4-25 words, assigned to ideas as the importer assigns them (at most 2 per idea and 6 in all), within the
57
37
  * per-entry budget (150 words, and the 10% source share with its 25-word floor).
58
38
  */
59
- export function publicEditorial(db, e) {
39
+ function publicEditorial(e, capture) {
60
40
  if (e.review.note !== 'accepted')
61
41
  return undefined;
62
42
  const o = e.output, [first] = firstSentence(o.gist ?? ''), lead = first && first.length <= L.leadChars ? first : undefined;
63
- const ideas = (o.ideas ?? []).map(i => ({ claim: i.claim, detail: i.detail, quotes: [] }));
43
+ const ideas = (o.ideas ?? []).map((i) => ({ claim: i.claim, detail: i.detail, quotes: [] }));
64
44
  if (ideas.length && o.quotes?.length) {
65
- const source = markdownFor(db, e.revisionId), budget = Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare))), taken = [], texts = [];
45
+ const source = capture(), budget = Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare))), taken = [], texts = [];
66
46
  for (const q of o.quotes) {
67
- const match = locateAll(source, q.text).find(m => !taken.some(t => m.start < t.end && t.start < m.end));
47
+ const match = locateAll(source, q.text).find((m) => !taken.some((t) => m.start < t.end && t.start < m.end));
68
48
  if (!match)
69
49
  continue;
70
50
  const text = source.slice(match.start, match.end), n = words(text);
@@ -74,7 +54,7 @@ export function publicEditorial(db, e) {
74
54
  }
75
55
  }
76
56
  let used = 0, count = 0;
77
- assignQuotes(ideas, texts.map(text => ({ text }))).forEach((idea, k) => {
57
+ assignQuotes(ideas, texts.map((text) => ({ text }))).forEach((idea, k) => {
78
58
  const n = words(texts[k]);
79
59
  if (idea === null || count >= L.maxQuotes || used + n > budget)
80
60
  return;
@@ -85,3 +65,31 @@ export function publicEditorial(db, e) {
85
65
  }
86
66
  return lead || ideas.length ? { ...(lead ? { lead } : {}), ideas } : undefined;
87
67
  }
68
+ /** Editorial: the input hash is the note's own Markdown hash, so an edited note is a new object and an unchanged one a duplicate. */
69
+ export const editorialKind = {
70
+ kind: 'editorial',
71
+ schemaVersion: 1,
72
+ admit({ output: e }) {
73
+ if (e?.schemaVersion !== 1 ||
74
+ e.source !== 'jungle-reading' ||
75
+ ![e.form, e.slug, e.status, e.title, e.sourceUrl, e.capture].every((v) => str(v, 16384)) ||
76
+ !str(e.markdown, 4 * 1024 ** 2) ||
77
+ sha256(e.markdown) !== e.markdownSha256)
78
+ fail();
79
+ const output = canonicalEditorial(e);
80
+ if (Buffer.byteLength(output) > editorialBytes)
81
+ fail();
82
+ return { output, inputHash: e.markdownSha256, schemaVersion: 1 };
83
+ },
84
+ reviewPaths(d, status) {
85
+ if (status !== null && status !== 'accepted' && status !== 'rejected')
86
+ fail();
87
+ return (d.review.note ?? null) !== status ? [['review.note', status]] : [];
88
+ },
89
+ applyReview(_review, _output, path, value) {
90
+ if (path !== 'review.note')
91
+ fail();
92
+ return value ? { note: value } : {};
93
+ },
94
+ publish: publicEditorial,
95
+ };