@retinue/agentkit 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (405) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +310 -0
  3. package/dist/adapters/bullmq/consumer.d.ts +33 -0
  4. package/dist/adapters/bullmq/consumer.js +41 -0
  5. package/dist/adapters/bullmq/dispatcher.d.ts +74 -0
  6. package/dist/adapters/bullmq/dispatcher.js +160 -0
  7. package/dist/adapters/bullmq/export.d.ts +31 -0
  8. package/dist/adapters/bullmq/export.js +53 -0
  9. package/dist/adapters/bullmq/extraction.d.ts +42 -0
  10. package/dist/adapters/bullmq/extraction.js +63 -0
  11. package/dist/adapters/bullmq/index.d.ts +13 -0
  12. package/dist/adapters/bullmq/index.js +13 -0
  13. package/dist/adapters/bullmq/lock.d.ts +77 -0
  14. package/dist/adapters/bullmq/lock.js +126 -0
  15. package/dist/adapters/bullmq/queue.d.ts +50 -0
  16. package/dist/adapters/bullmq/queue.js +81 -0
  17. package/dist/adapters/memory/artifact-exports.d.ts +11 -0
  18. package/dist/adapters/memory/artifact-exports.js +102 -0
  19. package/dist/adapters/memory/artifacts.d.ts +15 -0
  20. package/dist/adapters/memory/artifacts.js +134 -0
  21. package/dist/adapters/memory/blobs.d.ts +7 -0
  22. package/dist/adapters/memory/blobs.js +27 -0
  23. package/dist/adapters/memory/evaluation.d.ts +18 -0
  24. package/dist/adapters/memory/evaluation.js +148 -0
  25. package/dist/adapters/memory/files.d.ts +27 -0
  26. package/dist/adapters/memory/files.js +0 -0
  27. package/dist/adapters/memory/flows.d.ts +16 -0
  28. package/dist/adapters/memory/flows.js +117 -0
  29. package/dist/adapters/memory/hitl.d.ts +9 -0
  30. package/dist/adapters/memory/hitl.js +130 -0
  31. package/dist/adapters/memory/idempotency.d.ts +13 -0
  32. package/dist/adapters/memory/idempotency.js +32 -0
  33. package/dist/adapters/memory/index.d.ts +39 -0
  34. package/dist/adapters/memory/index.js +107 -0
  35. package/dist/adapters/memory/knowledge.d.ts +43 -0
  36. package/dist/adapters/memory/knowledge.js +248 -0
  37. package/dist/adapters/memory/mcp.d.ts +9 -0
  38. package/dist/adapters/memory/mcp.js +37 -0
  39. package/dist/adapters/memory/message-store.d.ts +17 -0
  40. package/dist/adapters/memory/message-store.js +70 -0
  41. package/dist/adapters/memory/principal-memory.d.ts +7 -0
  42. package/dist/adapters/memory/principal-memory.js +83 -0
  43. package/dist/adapters/memory/runtime.d.ts +29 -0
  44. package/dist/adapters/memory/runtime.js +0 -0
  45. package/dist/adapters/memory/sessions.d.ts +29 -0
  46. package/dist/adapters/memory/sessions.js +0 -0
  47. package/dist/adapters/memory/skills.d.ts +10 -0
  48. package/dist/adapters/memory/skills.js +41 -0
  49. package/dist/adapters/memory/thread-summary.d.ts +7 -0
  50. package/dist/adapters/memory/thread-summary.js +29 -0
  51. package/dist/adapters/memory/usage-limits.d.ts +13 -0
  52. package/dist/adapters/memory/usage-limits.js +72 -0
  53. package/dist/adapters/memory/usage.d.ts +16 -0
  54. package/dist/adapters/memory/usage.js +279 -0
  55. package/dist/adapters/otel/index.d.ts +111 -0
  56. package/dist/adapters/otel/index.js +133 -0
  57. package/dist/adapters/postgres/artifact-exports.d.ts +12 -0
  58. package/dist/adapters/postgres/artifact-exports.js +117 -0
  59. package/dist/adapters/postgres/artifacts.d.ts +16 -0
  60. package/dist/adapters/postgres/artifacts.js +172 -0
  61. package/dist/adapters/postgres/checkpoint-store.d.ts +16 -0
  62. package/dist/adapters/postgres/checkpoint-store.js +34 -0
  63. package/dist/adapters/postgres/config.d.ts +15 -0
  64. package/dist/adapters/postgres/config.js +187 -0
  65. package/dist/adapters/postgres/conversation-store.d.ts +4 -0
  66. package/dist/adapters/postgres/conversation-store.js +82 -0
  67. package/dist/adapters/postgres/evaluation.d.ts +17 -0
  68. package/dist/adapters/postgres/evaluation.js +193 -0
  69. package/dist/adapters/postgres/file-content.d.ts +30 -0
  70. package/dist/adapters/postgres/file-content.js +111 -0
  71. package/dist/adapters/postgres/files.d.ts +19 -0
  72. package/dist/adapters/postgres/files.js +209 -0
  73. package/dist/adapters/postgres/flows.d.ts +20 -0
  74. package/dist/adapters/postgres/flows.js +206 -0
  75. package/dist/adapters/postgres/hitl.d.ts +5 -0
  76. package/dist/adapters/postgres/hitl.js +247 -0
  77. package/dist/adapters/postgres/index.d.ts +35 -0
  78. package/dist/adapters/postgres/index.js +35 -0
  79. package/dist/adapters/postgres/knowledge.d.ts +48 -0
  80. package/dist/adapters/postgres/knowledge.js +255 -0
  81. package/dist/adapters/postgres/memory.d.ts +14 -0
  82. package/dist/adapters/postgres/memory.js +194 -0
  83. package/dist/adapters/postgres/message-store.d.ts +11 -0
  84. package/dist/adapters/postgres/message-store.js +145 -0
  85. package/dist/adapters/postgres/migrations.d.ts +69 -0
  86. package/dist/adapters/postgres/migrations.js +1594 -0
  87. package/dist/adapters/postgres/pg-executor.d.ts +19 -0
  88. package/dist/adapters/postgres/pg-executor.js +32 -0
  89. package/dist/adapters/postgres/retention.d.ts +26 -0
  90. package/dist/adapters/postgres/retention.js +59 -0
  91. package/dist/adapters/postgres/rollups.d.ts +17 -0
  92. package/dist/adapters/postgres/rollups.js +267 -0
  93. package/dist/adapters/postgres/run-coordinator.d.ts +5 -0
  94. package/dist/adapters/postgres/run-coordinator.js +98 -0
  95. package/dist/adapters/postgres/run-event-log.d.ts +26 -0
  96. package/dist/adapters/postgres/run-event-log.js +30 -0
  97. package/dist/adapters/postgres/run-store.d.ts +4 -0
  98. package/dist/adapters/postgres/run-store.js +199 -0
  99. package/dist/adapters/postgres/schema.d.ts +39 -0
  100. package/dist/adapters/postgres/schema.js +70 -0
  101. package/dist/adapters/postgres/session-state.d.ts +7 -0
  102. package/dist/adapters/postgres/session-state.js +99 -0
  103. package/dist/adapters/postgres/sql.d.ts +8 -0
  104. package/dist/adapters/postgres/sql.js +2 -0
  105. package/dist/adapters/postgres/transaction.d.ts +37 -0
  106. package/dist/adapters/postgres/transaction.js +93 -0
  107. package/dist/adapters/postgres/unit-of-work.d.ts +18 -0
  108. package/dist/adapters/postgres/unit-of-work.js +8 -0
  109. package/dist/adapters/postgres/usage-limits.d.ts +15 -0
  110. package/dist/adapters/postgres/usage-limits.js +136 -0
  111. package/dist/adapters/postgres/usage.d.ts +15 -0
  112. package/dist/adapters/postgres/usage.js +226 -0
  113. package/dist/adapters/redis/index.d.ts +9 -0
  114. package/dist/adapters/redis/index.js +9 -0
  115. package/dist/adapters/redis/realtime.d.ts +74 -0
  116. package/dist/adapters/redis/realtime.js +112 -0
  117. package/dist/adapters/supabase/index.d.ts +88 -0
  118. package/dist/adapters/supabase/index.js +84 -0
  119. package/dist/adapters/supabase/realtime.d.ts +30 -0
  120. package/dist/adapters/supabase/realtime.js +53 -0
  121. package/dist/adapters/supabase/rls.d.ts +99 -0
  122. package/dist/adapters/supabase/rls.js +216 -0
  123. package/dist/adapters/supabase/storage.d.ts +50 -0
  124. package/dist/adapters/supabase/storage.js +207 -0
  125. package/dist/agents/agent.d.ts +66 -0
  126. package/dist/agents/agent.js +209 -0
  127. package/dist/agents/define.d.ts +21 -0
  128. package/dist/agents/define.js +63 -0
  129. package/dist/agents/engine.d.ts +98 -0
  130. package/dist/agents/engine.js +462 -0
  131. package/dist/agents/index.d.ts +50 -0
  132. package/dist/agents/index.js +17 -0
  133. package/dist/artifacts/index.d.ts +114 -0
  134. package/dist/artifacts/index.js +219 -0
  135. package/dist/authorization/index.d.ts +76 -0
  136. package/dist/authorization/index.js +76 -0
  137. package/dist/capabilities/index.d.ts +120 -0
  138. package/dist/capabilities/index.js +167 -0
  139. package/dist/capabilities/runtime.d.ts +89 -0
  140. package/dist/capabilities/runtime.js +84 -0
  141. package/dist/citations/index.d.ts +161 -0
  142. package/dist/citations/index.js +182 -0
  143. package/dist/context/assembler.d.ts +82 -0
  144. package/dist/context/assembler.js +129 -0
  145. package/dist/context/compaction.d.ts +45 -0
  146. package/dist/context/compaction.js +55 -0
  147. package/dist/context/index.d.ts +75 -0
  148. package/dist/context/index.js +17 -0
  149. package/dist/core/content-parts.d.ts +194 -0
  150. package/dist/core/content-parts.js +23 -0
  151. package/dist/core/context.d.ts +51 -0
  152. package/dist/core/context.js +9 -0
  153. package/dist/core/env.d.ts +25 -0
  154. package/dist/core/env.js +41 -0
  155. package/dist/core/errors.d.ts +30 -0
  156. package/dist/core/errors.js +65 -0
  157. package/dist/core/events.d.ts +139 -0
  158. package/dist/core/events.js +99 -0
  159. package/dist/core/ids.d.ts +52 -0
  160. package/dist/core/ids.js +13 -0
  161. package/dist/core/index.d.ts +9 -0
  162. package/dist/core/index.js +9 -0
  163. package/dist/core/tokens.d.ts +22 -0
  164. package/dist/core/tokens.js +22 -0
  165. package/dist/core/validation.d.ts +34 -0
  166. package/dist/core/validation.js +176 -0
  167. package/dist/documents/extraction.d.ts +121 -0
  168. package/dist/documents/extraction.js +293 -0
  169. package/dist/documents/index.d.ts +199 -0
  170. package/dist/documents/index.js +65 -0
  171. package/dist/documents/parsers/pdf.d.ts +47 -0
  172. package/dist/documents/parsers/pdf.js +508 -0
  173. package/dist/documents/parsers/text.d.ts +59 -0
  174. package/dist/documents/parsers/text.js +325 -0
  175. package/dist/documents/read-tool.d.ts +52 -0
  176. package/dist/documents/read-tool.js +109 -0
  177. package/dist/documents/render.d.ts +29 -0
  178. package/dist/documents/render.js +59 -0
  179. package/dist/documents/vision.d.ts +159 -0
  180. package/dist/documents/vision.js +214 -0
  181. package/dist/entries/adapters-bullmq.d.ts +8 -0
  182. package/dist/entries/adapters-bullmq.js +8 -0
  183. package/dist/entries/adapters-otel.d.ts +13 -0
  184. package/dist/entries/adapters-otel.js +13 -0
  185. package/dist/entries/adapters-postgres.d.ts +10 -0
  186. package/dist/entries/adapters-postgres.js +10 -0
  187. package/dist/entries/adapters-redis.d.ts +3 -0
  188. package/dist/entries/adapters-redis.js +3 -0
  189. package/dist/entries/context.d.ts +20 -0
  190. package/dist/entries/context.js +20 -0
  191. package/dist/entries/flows.d.ts +15 -0
  192. package/dist/entries/flows.js +15 -0
  193. package/dist/entries/hitl.d.ts +10 -0
  194. package/dist/entries/hitl.js +10 -0
  195. package/dist/entries/knowledge.d.ts +18 -0
  196. package/dist/entries/knowledge.js +19 -0
  197. package/dist/entries/mcp.d.ts +10 -0
  198. package/dist/entries/mcp.js +10 -0
  199. package/dist/entries/observability.d.ts +14 -0
  200. package/dist/entries/observability.js +16 -0
  201. package/dist/entries/persistence.d.ts +11 -0
  202. package/dist/entries/persistence.js +11 -0
  203. package/dist/entries/providers.d.ts +14 -0
  204. package/dist/entries/providers.js +14 -0
  205. package/dist/entries/runtime.d.ts +13 -0
  206. package/dist/entries/runtime.js +13 -0
  207. package/dist/entries/server.d.ts +24 -0
  208. package/dist/entries/server.js +24 -0
  209. package/dist/entries/tools.d.ts +21 -0
  210. package/dist/entries/tools.js +21 -0
  211. package/dist/entries/usage.d.ts +10 -0
  212. package/dist/entries/usage.js +10 -0
  213. package/dist/evaluation/gate.d.ts +168 -0
  214. package/dist/evaluation/gate.js +180 -0
  215. package/dist/evaluation/graders.d.ts +125 -0
  216. package/dist/evaluation/graders.js +203 -0
  217. package/dist/evaluation/index.d.ts +120 -0
  218. package/dist/evaluation/index.js +183 -0
  219. package/dist/evaluation/judge.d.ts +75 -0
  220. package/dist/evaluation/judge.js +111 -0
  221. package/dist/export/index.d.ts +162 -0
  222. package/dist/export/index.js +363 -0
  223. package/dist/export/markdown.d.ts +19 -0
  224. package/dist/export/markdown.js +29 -0
  225. package/dist/export/pdf.d.ts +73 -0
  226. package/dist/export/pdf.js +407 -0
  227. package/dist/files/context.d.ts +97 -0
  228. package/dist/files/context.js +185 -0
  229. package/dist/files/index.d.ts +210 -0
  230. package/dist/files/index.js +338 -0
  231. package/dist/files/read-tool.d.ts +81 -0
  232. package/dist/files/read-tool.js +163 -0
  233. package/dist/files/turn-parts.d.ts +96 -0
  234. package/dist/files/turn-parts.js +171 -0
  235. package/dist/flows/index.d.ts +270 -0
  236. package/dist/flows/index.js +62 -0
  237. package/dist/flows/interpreter.d.ts +146 -0
  238. package/dist/flows/interpreter.js +426 -0
  239. package/dist/flows/runner.d.ts +145 -0
  240. package/dist/flows/runner.js +270 -0
  241. package/dist/graphql/index.d.ts +8 -0
  242. package/dist/graphql/index.js +8 -0
  243. package/dist/graphql/resolvers.d.ts +237 -0
  244. package/dist/graphql/resolvers.js +253 -0
  245. package/dist/graphql/schema.d.ts +11 -0
  246. package/dist/graphql/schema.js +258 -0
  247. package/dist/graphql/sse.d.ts +77 -0
  248. package/dist/graphql/sse.js +100 -0
  249. package/dist/hitl/approved-execution.d.ts +127 -0
  250. package/dist/hitl/approved-execution.js +177 -0
  251. package/dist/hitl/index.d.ts +79 -0
  252. package/dist/hitl/index.js +12 -0
  253. package/dist/hitl/service.d.ts +221 -0
  254. package/dist/hitl/service.js +268 -0
  255. package/dist/idempotency/index.d.ts +70 -0
  256. package/dist/idempotency/index.js +59 -0
  257. package/dist/index.d.ts +103 -0
  258. package/dist/index.js +60 -0
  259. package/dist/knowledge/chunking.d.ts +57 -0
  260. package/dist/knowledge/chunking.js +158 -0
  261. package/dist/knowledge/index.d.ts +119 -0
  262. package/dist/knowledge/index.js +166 -0
  263. package/dist/knowledge/retrieval.d.ts +146 -0
  264. package/dist/knowledge/retrieval.js +170 -0
  265. package/dist/loadtest/harness.d.ts +168 -0
  266. package/dist/loadtest/harness.js +507 -0
  267. package/dist/loadtest/index.d.ts +13 -0
  268. package/dist/loadtest/index.js +13 -0
  269. package/dist/loadtest/injection.d.ts +89 -0
  270. package/dist/loadtest/injection.js +147 -0
  271. package/dist/loadtest/metrics.d.ts +197 -0
  272. package/dist/loadtest/metrics.js +160 -0
  273. package/dist/loadtest/runbooks.d.ts +28 -0
  274. package/dist/loadtest/runbooks.js +159 -0
  275. package/dist/loadtest/scenario.d.ts +104 -0
  276. package/dist/loadtest/scenario.js +208 -0
  277. package/dist/mcp/egress.d.ts +53 -0
  278. package/dist/mcp/egress.js +115 -0
  279. package/dist/mcp/index.d.ts +93 -0
  280. package/dist/mcp/index.js +33 -0
  281. package/dist/mcp/provider.d.ts +62 -0
  282. package/dist/mcp/provider.js +0 -0
  283. package/dist/models/index.d.ts +98 -0
  284. package/dist/models/index.js +74 -0
  285. package/dist/models/pricing.d.ts +24 -0
  286. package/dist/models/pricing.js +37 -0
  287. package/dist/models/provider-factory.d.ts +31 -0
  288. package/dist/models/provider-factory.js +67 -0
  289. package/dist/models/streaming.d.ts +145 -0
  290. package/dist/models/streaming.js +272 -0
  291. package/dist/models/vision.d.ts +38 -0
  292. package/dist/models/vision.js +62 -0
  293. package/dist/persistence/index.d.ts +1654 -0
  294. package/dist/persistence/index.js +226 -0
  295. package/dist/principal-memory/index.d.ts +106 -0
  296. package/dist/principal-memory/index.js +89 -0
  297. package/dist/retention/index.d.ts +89 -0
  298. package/dist/retention/index.js +70 -0
  299. package/dist/runtime/checkpoint.d.ts +37 -0
  300. package/dist/runtime/checkpoint.js +22 -0
  301. package/dist/runtime/index.d.ts +118 -0
  302. package/dist/runtime/index.js +69 -0
  303. package/dist/runtime/retry.d.ts +95 -0
  304. package/dist/runtime/retry.js +126 -0
  305. package/dist/runtime/serialization.d.ts +85 -0
  306. package/dist/runtime/serialization.js +95 -0
  307. package/dist/runtime/streaming.d.ts +54 -0
  308. package/dist/runtime/streaming.js +115 -0
  309. package/dist/runtime/worker.d.ts +130 -0
  310. package/dist/runtime/worker.js +405 -0
  311. package/dist/security/checklist.d.ts +53 -0
  312. package/dist/security/checklist.js +204 -0
  313. package/dist/security/findings.d.ts +56 -0
  314. package/dist/security/findings.js +168 -0
  315. package/dist/security/index.d.ts +14 -0
  316. package/dist/security/index.js +14 -0
  317. package/dist/security/prompt-safety.d.ts +100 -0
  318. package/dist/security/prompt-safety.js +133 -0
  319. package/dist/server/boot.d.ts +32 -0
  320. package/dist/server/boot.js +36 -0
  321. package/dist/server/cli-worker.d.ts +37 -0
  322. package/dist/server/cli-worker.js +151 -0
  323. package/dist/server/cli.d.ts +27 -0
  324. package/dist/server/cli.js +74 -0
  325. package/dist/server/config.d.ts +42 -0
  326. package/dist/server/config.js +127 -0
  327. package/dist/server/health.d.ts +59 -0
  328. package/dist/server/health.js +90 -0
  329. package/dist/server/host.d.ts +39 -0
  330. package/dist/server/host.js +124 -0
  331. package/dist/server/index.d.ts +15 -0
  332. package/dist/server/index.js +15 -0
  333. package/dist/server/main.d.ts +16 -0
  334. package/dist/server/main.js +31 -0
  335. package/dist/server/sse-route.d.ts +21 -0
  336. package/dist/server/sse-route.js +282 -0
  337. package/dist/skills/index.d.ts +67 -0
  338. package/dist/skills/index.js +31 -0
  339. package/dist/skills/resolver.d.ts +54 -0
  340. package/dist/skills/resolver.js +121 -0
  341. package/dist/teams/index.d.ts +93 -0
  342. package/dist/teams/index.js +207 -0
  343. package/dist/telemetry/index.d.ts +157 -0
  344. package/dist/telemetry/index.js +71 -0
  345. package/dist/telemetry/instrument.d.ts +108 -0
  346. package/dist/telemetry/instrument.js +232 -0
  347. package/dist/telemetry/log-events.d.ts +17 -0
  348. package/dist/telemetry/log-events.js +58 -0
  349. package/dist/telemetry/metrics.d.ts +123 -0
  350. package/dist/telemetry/metrics.js +135 -0
  351. package/dist/telemetry/noop.d.ts +39 -0
  352. package/dist/telemetry/noop.js +143 -0
  353. package/dist/telemetry/redaction.d.ts +64 -0
  354. package/dist/telemetry/redaction.js +153 -0
  355. package/dist/telemetry/spans.d.ts +56 -0
  356. package/dist/telemetry/spans.js +78 -0
  357. package/dist/telemetry/trace-context.d.ts +55 -0
  358. package/dist/telemetry/trace-context.js +60 -0
  359. package/dist/toolkit/compute.d.ts +53 -0
  360. package/dist/toolkit/compute.js +152 -0
  361. package/dist/toolkit/data.d.ts +98 -0
  362. package/dist/toolkit/data.js +235 -0
  363. package/dist/toolkit/http.d.ts +113 -0
  364. package/dist/toolkit/http.js +205 -0
  365. package/dist/toolkit/index.d.ts +21 -0
  366. package/dist/toolkit/index.js +17 -0
  367. package/dist/toolkit/web.d.ts +107 -0
  368. package/dist/toolkit/web.js +147 -0
  369. package/dist/tools/define.d.ts +25 -0
  370. package/dist/tools/define.js +45 -0
  371. package/dist/tools/delegating.d.ts +132 -0
  372. package/dist/tools/delegating.js +211 -0
  373. package/dist/tools/index.d.ts +129 -0
  374. package/dist/tools/index.js +33 -0
  375. package/dist/tools/library/compute.d.ts +11 -0
  376. package/dist/tools/library/compute.js +46 -0
  377. package/dist/tools/library/data.d.ts +16 -0
  378. package/dist/tools/library/data.js +92 -0
  379. package/dist/tools/library/http.d.ts +28 -0
  380. package/dist/tools/library/http.js +71 -0
  381. package/dist/tools/library/index.d.ts +97 -0
  382. package/dist/tools/library/index.js +134 -0
  383. package/dist/tools/library/knowledge.d.ts +39 -0
  384. package/dist/tools/library/knowledge.js +58 -0
  385. package/dist/tools/library/web.d.ts +19 -0
  386. package/dist/tools/library/web.js +65 -0
  387. package/dist/tools/meta-tools.d.ts +19 -0
  388. package/dist/tools/meta-tools.js +36 -0
  389. package/dist/tools/registry.d.ts +146 -0
  390. package/dist/tools/registry.js +291 -0
  391. package/dist/usage/index.d.ts +105 -0
  392. package/dist/usage/index.js +20 -0
  393. package/dist/usage/quota.d.ts +258 -0
  394. package/dist/usage/quota.js +510 -0
  395. package/dist/usage/recorder.d.ts +29 -0
  396. package/dist/usage/recorder.js +96 -0
  397. package/dist/usage/rollups.d.ts +121 -0
  398. package/dist/usage/rollups.js +157 -0
  399. package/dist/worker/export.d.ts +57 -0
  400. package/dist/worker/export.js +81 -0
  401. package/dist/worker/extraction.d.ts +57 -0
  402. package/dist/worker/extraction.js +84 -0
  403. package/dist/worker/main.d.ts +103 -0
  404. package/dist/worker/main.js +159 -0
  405. package/package.json +187 -0
@@ -0,0 +1,121 @@
1
+ /**
2
+ * The rollup job and the reconciliation report (#139).
3
+ *
4
+ * `UsageRollupStore.rebuild` does the arithmetic. What lives here is *which buckets to rebuild and when*, and
5
+ * the comparison against what the provider says we spent.
6
+ *
7
+ * **The work list is derived from the ledger.** `listStaleBuckets` asks which buckets have events newer than
8
+ * their last computation, so an interrupted job resumes by asking again rather than by remembering where it
9
+ * was. The same shape #135's re-indexing uses, and for the same reason: a cursor that has to be persisted is a
10
+ * cursor that can be lost, and a lost rollup cursor is silently wrong totals.
11
+ *
12
+ * **Reconciliation reports, it does not correct.** A discrepancy between our ledger and a provider's invoice
13
+ * has several causes — a rounding difference, an event we never recorded, a charge for a call we did not make,
14
+ * a provider restatement — and they want different responses. A job that "corrected" the ledger would erase
15
+ * the evidence needed to tell them apart, and the ledger is append-only precisely so that evidence survives.
16
+ */
17
+ import type { ExecutionContext } from "../core/context.js";
18
+ import type { TenantId } from "../core/ids.js";
19
+ import type { RollupPeriod, UsageRollup, UsageRollupStore } from "../persistence/index.js";
20
+ export type RollupJobDeps = {
21
+ readonly rollups: UsageRollupStore;
22
+ readonly log?: (message: string, detail?: Readonly<Record<string, unknown>>) => void;
23
+ };
24
+ export declare const createRollupJob: (deps: RollupJobDeps) => {
25
+ /**
26
+ * Rebuild one page of stale buckets — AC-4.
27
+ *
28
+ * A page at a time, so an interruption loses at most a page and no bookkeeping. Rebuilding is a
29
+ * *recomputation*, so re-running a bucket cannot double count and two workers racing one bucket write the
30
+ * same value — which is what makes "safe under concurrency" a property of the design rather than a lock.
31
+ */
32
+ run(context: {
33
+ readonly tenantId: TenantId;
34
+ }, input: {
35
+ readonly period: RollupPeriod;
36
+ readonly since: string;
37
+ readonly limit: number;
38
+ }): Promise<{
39
+ readonly rebuilt: number;
40
+ readonly remaining: number;
41
+ }>;
42
+ /** Rebuild a specific range, for a backfill or after a correcting event. */
43
+ rebuildRange(context: {
44
+ readonly tenantId: TenantId;
45
+ }, input: {
46
+ readonly period: RollupPeriod;
47
+ readonly from: string;
48
+ readonly to: string;
49
+ }): Promise<readonly UsageRollup[]>;
50
+ };
51
+ export type RollupJob = ReturnType<typeof createRollupJob>;
52
+ /**
53
+ * What the provider says a period cost.
54
+ *
55
+ * `costMinorUnits` in the same integer minor units the ledger uses, so a comparison is exact arithmetic rather
56
+ * than a float difference that is always slightly nonzero.
57
+ */
58
+ export type ProviderStatement = {
59
+ readonly provider: string;
60
+ readonly from: string;
61
+ readonly to: string;
62
+ readonly costMinorUnits: number;
63
+ readonly currency: string;
64
+ };
65
+ /**
66
+ * The tolerance a discrepancy has to clear to be reported — AC-5.
67
+ *
68
+ * **Both a fraction and a floor**, and both are needed. A fraction alone reports every tiny period as broken:
69
+ * on a €0.02 hour a one-cent rounding difference is 50%. A floor alone stops scaling: on a €10,000 month a
70
+ * €5 absolute tolerance is noise nobody can act on. A discrepancy must exceed *both* to be worth someone's
71
+ * attention.
72
+ */
73
+ export declare const DEFAULT_RECONCILIATION_TOLERANCE: {
74
+ readonly fraction: 0.02;
75
+ readonly floorMinorUnits: 100;
76
+ };
77
+ export type ReconciliationTolerance = {
78
+ readonly fraction: number;
79
+ readonly floorMinorUnits: number;
80
+ };
81
+ /** Why a period did not reconcile. Separate values because each points at a different investigation. */
82
+ export declare const DISCREPANCY_KINDS: readonly ["under-recorded", "over-recorded", "currency-mismatch"];
83
+ export type DiscrepancyKind = (typeof DISCREPANCY_KINDS)[number];
84
+ /**
85
+ * Named for usage specifically, because `files/index.ts` has a `ReconciliationReport` of its own — for orphaned
86
+ * objects, which is a different problem with a different report. The package barrel refused both, which is how
87
+ * the collision surfaced.
88
+ */
89
+ export type UsageReconciliationReport = {
90
+ readonly provider: string;
91
+ readonly from: string;
92
+ readonly to: string;
93
+ readonly ledgerMinorUnits: number;
94
+ readonly providerMinorUnits: number;
95
+ readonly deltaMinorUnits: number;
96
+ readonly withinTolerance: boolean;
97
+ readonly kind?: DiscrepancyKind;
98
+ readonly message?: string;
99
+ };
100
+ /**
101
+ * Compare the ledger's rollups against a provider statement.
102
+ *
103
+ * Reads the *rollups*, not the ledger, so reconciliation costs the same whether a tenant has spent a euro or a
104
+ * million — and so a discrepancy in the rollups themselves is visible rather than hidden by recomputing the
105
+ * thing being checked.
106
+ */
107
+ export declare const reconcileUsage: (context: ExecutionContext, deps: {
108
+ readonly rollups: UsageRollupStore;
109
+ }, input: {
110
+ readonly statement: ProviderStatement;
111
+ /**
112
+ * The currency the ledger records in.
113
+ *
114
+ * Passed rather than read from a rollup, because a range with no usage has no rollup to read it from — and
115
+ * "no usage" is exactly when a statement showing a charge matters most.
116
+ */
117
+ readonly ledgerCurrency: string;
118
+ readonly period?: RollupPeriod;
119
+ readonly tolerance?: ReconciliationTolerance;
120
+ }) => Promise<UsageReconciliationReport>;
121
+ //# sourceMappingURL=rollups.d.ts.map
@@ -0,0 +1,157 @@
1
+ /**
2
+ * The rollup job and the reconciliation report (#139).
3
+ *
4
+ * `UsageRollupStore.rebuild` does the arithmetic. What lives here is *which buckets to rebuild and when*, and
5
+ * the comparison against what the provider says we spent.
6
+ *
7
+ * **The work list is derived from the ledger.** `listStaleBuckets` asks which buckets have events newer than
8
+ * their last computation, so an interrupted job resumes by asking again rather than by remembering where it
9
+ * was. The same shape #135's re-indexing uses, and for the same reason: a cursor that has to be persisted is a
10
+ * cursor that can be lost, and a lost rollup cursor is silently wrong totals.
11
+ *
12
+ * **Reconciliation reports, it does not correct.** A discrepancy between our ledger and a provider's invoice
13
+ * has several causes — a rounding difference, an event we never recorded, a charge for a call we did not make,
14
+ * a provider restatement — and they want different responses. A job that "corrected" the ledger would erase
15
+ * the evidence needed to tell them apart, and the ledger is append-only precisely so that evidence survives.
16
+ */
17
+ import { bucketStartFor, bucketsBetween, nextBucket } from "./quota.js";
18
+ export const createRollupJob = (deps) => {
19
+ const log = deps.log ?? (() => { });
20
+ return {
21
+ /**
22
+ * Rebuild one page of stale buckets — AC-4.
23
+ *
24
+ * A page at a time, so an interruption loses at most a page and no bookkeeping. Rebuilding is a
25
+ * *recomputation*, so re-running a bucket cannot double count and two workers racing one bucket write the
26
+ * same value — which is what makes "safe under concurrency" a property of the design rather than a lock.
27
+ */
28
+ async run(context, input) {
29
+ const stale = await deps.rollups.listStaleBuckets({
30
+ tenantId: context.tenantId,
31
+ period: input.period,
32
+ since: input.since,
33
+ limit: input.limit,
34
+ });
35
+ for (const bucket of stale.items) {
36
+ // No `computedAt`: the store stamps it from its own clock, because staleness compares that stamp
37
+ // against when events were recorded and both must come from one source.
38
+ await deps.rollups.rebuild({
39
+ tenantId: context.tenantId,
40
+ period: bucket.period,
41
+ bucketStart: bucket.bucketStart,
42
+ /**
43
+ * The grain the store reported — #175.
44
+ *
45
+ * Dropping it would rebuild the tenant row twice and never build a principal's at all, so every
46
+ * per-person figure would read zero and every per-person quota would be unenforceable. The store
47
+ * reports both grains from one ledger pass precisely so the job does not have to know which
48
+ * principals were active.
49
+ */
50
+ ...(bucket.principalId === undefined ? {} : { principalId: bucket.principalId }),
51
+ });
52
+ }
53
+ // Asked again rather than computed, because events may have landed while this page ran — and a stale
54
+ // "remaining" is how a loop stops early or never stops.
55
+ const after = await deps.rollups.listStaleBuckets({
56
+ tenantId: context.tenantId,
57
+ period: input.period,
58
+ since: input.since,
59
+ limit: 1,
60
+ });
61
+ if (stale.items.length > 0)
62
+ log("rebuilt usage buckets", { period: input.period, count: stale.items.length });
63
+ return { rebuilt: stale.items.length, remaining: after.items.length };
64
+ },
65
+ /** Rebuild a specific range, for a backfill or after a correcting event. */
66
+ async rebuildRange(context, input) {
67
+ const written = [];
68
+ for (const bucketStart of bucketsBetween(input.period, input.from, input.to)) {
69
+ written.push(await deps.rollups.rebuild({
70
+ tenantId: context.tenantId,
71
+ period: input.period,
72
+ bucketStart,
73
+ }));
74
+ }
75
+ return written;
76
+ },
77
+ };
78
+ };
79
+ /**
80
+ * The tolerance a discrepancy has to clear to be reported — AC-5.
81
+ *
82
+ * **Both a fraction and a floor**, and both are needed. A fraction alone reports every tiny period as broken:
83
+ * on a €0.02 hour a one-cent rounding difference is 50%. A floor alone stops scaling: on a €10,000 month a
84
+ * €5 absolute tolerance is noise nobody can act on. A discrepancy must exceed *both* to be worth someone's
85
+ * attention.
86
+ */
87
+ export const DEFAULT_RECONCILIATION_TOLERANCE = { fraction: 0.02, floorMinorUnits: 100 };
88
+ /** Why a period did not reconcile. Separate values because each points at a different investigation. */
89
+ export const DISCREPANCY_KINDS = ["under-recorded", "over-recorded", "currency-mismatch"];
90
+ /**
91
+ * Compare the ledger's rollups against a provider statement.
92
+ *
93
+ * Reads the *rollups*, not the ledger, so reconciliation costs the same whether a tenant has spent a euro or a
94
+ * million — and so a discrepancy in the rollups themselves is visible rather than hidden by recomputing the
95
+ * thing being checked.
96
+ */
97
+ export const reconcileUsage = async (context, deps, input) => {
98
+ const period = input.period ?? "day";
99
+ const tolerance = input.tolerance ?? DEFAULT_RECONCILIATION_TOLERANCE;
100
+ const ledger = await deps.rollups.sum({
101
+ tenantId: context.tenantId,
102
+ period,
103
+ from: bucketStartFor(period, input.statement.from),
104
+ // Exclusive upper bound, extended to the end of the bucket containing `to` — a statement ending mid-day
105
+ // still covers that day's spend, and truncating would report every statement as under-recorded.
106
+ to: nextBucket(period, bucketStartFor(period, input.statement.to)),
107
+ });
108
+ // Currency first, because comparing two numbers in different currencies is not a discrepancy — it is a
109
+ // meaningless subtraction that would report a plausible figure. Checked before the arithmetic rather than
110
+ // after, so no number is ever computed across currencies.
111
+ if (input.ledgerCurrency !== input.statement.currency)
112
+ return {
113
+ provider: input.statement.provider,
114
+ from: input.statement.from,
115
+ to: input.statement.to,
116
+ ledgerMinorUnits: ledger.costMinorUnits,
117
+ providerMinorUnits: input.statement.costMinorUnits,
118
+ // Deliberately zero rather than a difference: there is no difference between amounts in different
119
+ // currencies, and a number here would be quoted as one.
120
+ deltaMinorUnits: 0,
121
+ withinTolerance: false,
122
+ kind: "currency-mismatch",
123
+ message: `The ledger records ${input.ledgerCurrency} and ${input.statement.provider} billed in ${input.statement.currency}; the totals cannot be compared.`,
124
+ };
125
+ const delta = ledger.costMinorUnits - input.statement.costMinorUnits;
126
+ const magnitude = Math.abs(delta);
127
+ const base = Math.max(input.statement.costMinorUnits, 1);
128
+ // Both thresholds, not either: a fraction alone flags every tiny period, a floor alone stops scaling.
129
+ const withinTolerance = magnitude <= tolerance.floorMinorUnits || magnitude / base <= tolerance.fraction;
130
+ if (withinTolerance)
131
+ return {
132
+ provider: input.statement.provider,
133
+ from: input.statement.from,
134
+ to: input.statement.to,
135
+ ledgerMinorUnits: ledger.costMinorUnits,
136
+ providerMinorUnits: input.statement.costMinorUnits,
137
+ deltaMinorUnits: delta,
138
+ withinTolerance: true,
139
+ };
140
+ const kind = delta < 0 ? "under-recorded" : "over-recorded";
141
+ return {
142
+ provider: input.statement.provider,
143
+ from: input.statement.from,
144
+ to: input.statement.to,
145
+ ledgerMinorUnits: ledger.costMinorUnits,
146
+ providerMinorUnits: input.statement.costMinorUnits,
147
+ deltaMinorUnits: delta,
148
+ withinTolerance: false,
149
+ kind,
150
+ // Both figures and the direction, because "under-recorded" and "over-recorded" point at different
151
+ // investigations: the first is usage we are not billing for, the second is a charge we cannot account for.
152
+ message: kind === "under-recorded"
153
+ ? `${input.statement.provider} billed ${input.statement.costMinorUnits} but the ledger records ${ledger.costMinorUnits} — ${magnitude} minor units of usage were not recorded.`
154
+ : `The ledger records ${ledger.costMinorUnits} but ${input.statement.provider} billed ${input.statement.costMinorUnits} — ${magnitude} minor units were recorded that the provider did not bill.`,
155
+ };
156
+ };
157
+ //# sourceMappingURL=rollups.js.map
@@ -0,0 +1,57 @@
1
+ /**
2
+ * The export worker (#134).
3
+ *
4
+ * Composition only, like the extraction worker — every bound and every failure decision lives in
5
+ * `export/index.ts`. What this owns is the loop.
6
+ *
7
+ * **A failed render completes the job.** The service records a typed failure and returns; retrying a
8
+ * document that cannot be rendered produces the same answer at the same cost forever. Only an infrastructure
9
+ * failure throws, and only that is worth retrying.
10
+ *
11
+ * The job carries a tenant but not a *principal*, so the host supplies the `ExecutionContext` the render runs
12
+ * as. That is deliberate: an export must be rendered with the entitlement of the person who asked for it, and
13
+ * a worker inventing an all-powerful context would make AC-5 decorative.
14
+ */
15
+ import type { ExecutionContext } from "../core/context.js";
16
+ import type { ArtifactExport } from "../persistence/index.js";
17
+ import type { ExportJob, ExportService } from "../export/index.js";
18
+ export interface ExportConsumer {
19
+ start(handler: (job: ExportJob) => Promise<void>): Promise<void> | void;
20
+ stop(graceMs: number): Promise<void>;
21
+ }
22
+ export type ExportWorkerConfig = {
23
+ /** Low, like extraction's: rendering is CPU-bound and four concurrent renders on one loop is four slow ones. */
24
+ readonly concurrency: number;
25
+ readonly shutdownGraceMs: number;
26
+ };
27
+ export declare const DEFAULT_EXPORT_WORKER_CONFIG: ExportWorkerConfig;
28
+ export type ExportWorkerDeps = {
29
+ readonly exports: ExportService;
30
+ readonly consumer: ExportConsumer;
31
+ /**
32
+ * The context a job renders as.
33
+ *
34
+ * Required, and required to be per-job: the render re-reads the artifact through `ArtifactService`, so this
35
+ * is what makes an export carry the requester's entitlement rather than the worker's.
36
+ */
37
+ readonly contextFor: (job: ExportJob) => Promise<ExecutionContext> | ExecutionContext;
38
+ readonly config?: Partial<ExportWorkerConfig>;
39
+ readonly log?: (message: string, detail?: Readonly<Record<string, unknown>>) => void;
40
+ readonly onOutcome?: (job: ExportJob, result: ArtifactExport) => void;
41
+ readonly onError?: (job: ExportJob, error: unknown) => void;
42
+ };
43
+ export declare const createExportWorker: (deps: ExportWorkerDeps) => {
44
+ start(): Promise<void>;
45
+ shutdown(reason: string): Promise<{
46
+ readonly graceful: boolean;
47
+ }>;
48
+ status(): {
49
+ running: boolean;
50
+ inFlight: number;
51
+ processed: number;
52
+ failed: number;
53
+ };
54
+ concurrency: number;
55
+ };
56
+ export type ExportWorker = ReturnType<typeof createExportWorker>;
57
+ //# sourceMappingURL=export.d.ts.map
@@ -0,0 +1,81 @@
1
+ /**
2
+ * The export worker (#134).
3
+ *
4
+ * Composition only, like the extraction worker — every bound and every failure decision lives in
5
+ * `export/index.ts`. What this owns is the loop.
6
+ *
7
+ * **A failed render completes the job.** The service records a typed failure and returns; retrying a
8
+ * document that cannot be rendered produces the same answer at the same cost forever. Only an infrastructure
9
+ * failure throws, and only that is worth retrying.
10
+ *
11
+ * The job carries a tenant but not a *principal*, so the host supplies the `ExecutionContext` the render runs
12
+ * as. That is deliberate: an export must be rendered with the entitlement of the person who asked for it, and
13
+ * a worker inventing an all-powerful context would make AC-5 decorative.
14
+ */
15
+ export const DEFAULT_EXPORT_WORKER_CONFIG = { concurrency: 2, shutdownGraceMs: 20_000 };
16
+ export const createExportWorker = (deps) => {
17
+ const config = { ...DEFAULT_EXPORT_WORKER_CONFIG, ...deps.config };
18
+ const log = deps.log ?? (() => { });
19
+ let running = false;
20
+ let inFlight = 0;
21
+ let processed = 0;
22
+ let failed = 0;
23
+ const settled = new Set();
24
+ const handle = async (job) => {
25
+ inFlight += 1;
26
+ try {
27
+ const context = await deps.contextFor(job);
28
+ const result = await deps.exports.render(job, context);
29
+ processed += 1;
30
+ // Counted, not thrown: a document that cannot be rendered is an outcome, and the queue must not retry it.
31
+ if (result.state === "failed")
32
+ failed += 1;
33
+ deps.onOutcome?.(job, result);
34
+ }
35
+ catch (error) {
36
+ // Only infrastructure reaches here — the service turns every render problem into a record. Rethrown so
37
+ // the queue does retry, which is right for a store that was briefly unreachable.
38
+ log("export job failed", { job, error });
39
+ deps.onError?.(job, error);
40
+ throw error;
41
+ }
42
+ finally {
43
+ inFlight -= 1;
44
+ }
45
+ };
46
+ return {
47
+ async start() {
48
+ running = true;
49
+ await deps.consumer.start(async (job) => {
50
+ const promise = handle(job);
51
+ settled.add(promise);
52
+ try {
53
+ await promise;
54
+ }
55
+ finally {
56
+ settled.delete(promise);
57
+ }
58
+ });
59
+ },
60
+ async shutdown(reason) {
61
+ log("export worker shutting down", { reason, inFlight });
62
+ running = false;
63
+ // Stop accepting first, then wait. Draining first would accept a job it has no intention of finishing.
64
+ await deps.consumer.stop(config.shutdownGraceMs);
65
+ const graceful = await Promise.race([
66
+ Promise.allSettled([...settled]).then(() => true),
67
+ new Promise((resolve) => {
68
+ const timer = setTimeout(() => resolve(false), config.shutdownGraceMs);
69
+ // Unref so a clean shutdown is not held open by its own deadline.
70
+ timer.unref?.();
71
+ }),
72
+ ]);
73
+ return { graceful };
74
+ },
75
+ status() {
76
+ return { running, inFlight, processed, failed };
77
+ },
78
+ concurrency: config.concurrency,
79
+ };
80
+ };
81
+ //# sourceMappingURL=export.js.map
@@ -0,0 +1,57 @@
1
+ /**
2
+ * The extraction worker (#131, AC-2 and AC-3).
3
+ *
4
+ * Composition only, like `worker/main.ts` — every bound and every failure decision lives in
5
+ * `documents/extraction.ts`, and duplicating any of them here would mean two places to look when a document
6
+ * misbehaves. What this file owns is the loop: claim, extract, report, and stop cleanly.
7
+ *
8
+ * Written against the same `JobConsumer`-shaped seam as the run worker so the ordering decisions — the part
9
+ * that can actually be wrong — are testable without Redis.
10
+ *
11
+ * **A failed extraction is not a failed job.** The pipeline records a typed failure and returns; the job
12
+ * completes. That is deliberate: a job that failed would be retried by the queue, and retrying a scan with no
13
+ * text layer produces the same answer at the same cost forever. Only an *infrastructure* failure — the store
14
+ * unreachable — throws, and only that is worth retrying.
15
+ */
16
+ import type { FileExtraction } from "../persistence/index.js";
17
+ import type { ExtractionJob } from "../documents/index.js";
18
+ import type { ExtractionService } from "../documents/extraction.js";
19
+ /** The consumer side. Mirrors `JobConsumer` for runs; `stop` must stop accepting before it waits. */
20
+ export interface ExtractionConsumer {
21
+ start(handler: (job: ExtractionJob) => Promise<void>): Promise<void> | void;
22
+ stop(graceMs: number): Promise<void>;
23
+ }
24
+ export type ExtractionWorkerConfig = {
25
+ /**
26
+ * Jobs at once. Low by default, and lower than the run worker's on purpose: extraction is CPU-bound, and
27
+ * four concurrent PDF parses on one event loop is four slow parses rather than four fast ones.
28
+ */
29
+ readonly concurrency: number;
30
+ readonly shutdownGraceMs: number;
31
+ };
32
+ export declare const DEFAULT_EXTRACTION_WORKER_CONFIG: ExtractionWorkerConfig;
33
+ export type ExtractionWorkerDeps = {
34
+ readonly extraction: ExtractionService;
35
+ readonly consumer: ExtractionConsumer;
36
+ readonly config?: Partial<ExtractionWorkerConfig>;
37
+ readonly log?: (message: string, detail?: Readonly<Record<string, unknown>>) => void;
38
+ /** Observability hook, and what the tests assert on. */
39
+ readonly onOutcome?: (job: ExtractionJob, extraction: FileExtraction) => void;
40
+ readonly onError?: (job: ExtractionJob, error: unknown) => void;
41
+ };
42
+ export type ExtractionWorkerStatus = {
43
+ readonly running: boolean;
44
+ readonly inFlight: number;
45
+ readonly processed: number;
46
+ readonly failed: number;
47
+ };
48
+ export declare const createExtractionWorker: (deps: ExtractionWorkerDeps) => {
49
+ start(): Promise<void>;
50
+ shutdown(reason: string): Promise<{
51
+ readonly graceful: boolean;
52
+ }>;
53
+ status(): ExtractionWorkerStatus;
54
+ concurrency: number;
55
+ };
56
+ export type ExtractionWorker = ReturnType<typeof createExtractionWorker>;
57
+ //# sourceMappingURL=extraction.d.ts.map
@@ -0,0 +1,84 @@
1
+ /**
2
+ * The extraction worker (#131, AC-2 and AC-3).
3
+ *
4
+ * Composition only, like `worker/main.ts` — every bound and every failure decision lives in
5
+ * `documents/extraction.ts`, and duplicating any of them here would mean two places to look when a document
6
+ * misbehaves. What this file owns is the loop: claim, extract, report, and stop cleanly.
7
+ *
8
+ * Written against the same `JobConsumer`-shaped seam as the run worker so the ordering decisions — the part
9
+ * that can actually be wrong — are testable without Redis.
10
+ *
11
+ * **A failed extraction is not a failed job.** The pipeline records a typed failure and returns; the job
12
+ * completes. That is deliberate: a job that failed would be retried by the queue, and retrying a scan with no
13
+ * text layer produces the same answer at the same cost forever. Only an *infrastructure* failure — the store
14
+ * unreachable — throws, and only that is worth retrying.
15
+ */
16
+ export const DEFAULT_EXTRACTION_WORKER_CONFIG = {
17
+ concurrency: 2,
18
+ shutdownGraceMs: 20_000,
19
+ };
20
+ export const createExtractionWorker = (deps) => {
21
+ const config = { ...DEFAULT_EXTRACTION_WORKER_CONFIG, ...deps.config };
22
+ const log = deps.log ?? (() => { });
23
+ let running = false;
24
+ let inFlight = 0;
25
+ let processed = 0;
26
+ let failed = 0;
27
+ const settled = new Set();
28
+ const handle = async (job) => {
29
+ inFlight += 1;
30
+ try {
31
+ const extraction = await deps.extraction.extract(job);
32
+ processed += 1;
33
+ // Counted, not thrown. A document that cannot be read is an outcome; the queue must not retry it.
34
+ if (extraction.state === "failed")
35
+ failed += 1;
36
+ deps.onOutcome?.(job, extraction);
37
+ }
38
+ catch (error) {
39
+ // Only infrastructure reaches here — the pipeline turns every document problem into a record. Rethrown
40
+ // so the queue *does* retry, which is right for a store that was briefly unreachable.
41
+ log("extraction job failed", { job, error });
42
+ deps.onError?.(job, error);
43
+ throw error;
44
+ }
45
+ finally {
46
+ inFlight -= 1;
47
+ }
48
+ };
49
+ return {
50
+ async start() {
51
+ running = true;
52
+ await deps.consumer.start(async (job) => {
53
+ const promise = handle(job);
54
+ settled.add(promise);
55
+ try {
56
+ await promise;
57
+ }
58
+ finally {
59
+ settled.delete(promise);
60
+ }
61
+ });
62
+ },
63
+ async shutdown(reason) {
64
+ log("extraction worker shutting down", { reason, inFlight });
65
+ running = false;
66
+ // Stop accepting first, then wait. Draining first would accept a job with no intention of finishing it.
67
+ await deps.consumer.stop(config.shutdownGraceMs);
68
+ const graceful = await Promise.race([
69
+ Promise.allSettled([...settled]).then(() => true),
70
+ new Promise((resolve) => {
71
+ const timer = setTimeout(() => resolve(false), config.shutdownGraceMs);
72
+ // Unref so a clean shutdown is not held open by its own deadline.
73
+ timer.unref?.();
74
+ }),
75
+ ]);
76
+ return { graceful };
77
+ },
78
+ status() {
79
+ return { running, inFlight, processed, failed };
80
+ },
81
+ concurrency: config.concurrency,
82
+ };
83
+ };
84
+ //# sourceMappingURL=extraction.js.map
@@ -0,0 +1,103 @@
1
+ /**
2
+ * The worker process entrypoint (#107).
3
+ *
4
+ * `runtime/worker.ts` has had the claim, heartbeat, checkpoint and recovery logic all along and
5
+ * nothing ran it. This is that missing process, and it is **composition only** — every recovery and
6
+ * heartbeat behaviour it relies on already exists and is already tested. Adding execution logic here
7
+ * would mean two places to look when a run misbehaves.
8
+ *
9
+ * Written against a `JobConsumer` seam rather than BullMQ directly, so the entrypoint's ordering
10
+ * decisions — which are the part that can actually be wrong — are testable without Redis.
11
+ */
12
+ import type { JobDispatcher, Run } from "../runtime/index.js";
13
+ import type { DurableWorker, ProcessResult } from "../runtime/worker.js";
14
+ import type { RunId, TenantId } from "../core/ids.js";
15
+ /**
16
+ * A unit of work off the queue. Mirrors the dispatcher's `enqueueRun` input.
17
+ *
18
+ * The trace fields (#143) are optional and unused by the runtime itself — `instrumentConsumer` reads them. They
19
+ * live on this type rather than on a parallel one so the consumer seam stays a single shape; a second type would
20
+ * mean an adapter deciding which to produce.
21
+ */
22
+ export type RunJob = {
23
+ readonly tenantId: TenantId;
24
+ readonly runId: RunId;
25
+ readonly traceparent?: string;
26
+ readonly enqueuedAt?: string;
27
+ };
28
+ /**
29
+ * The queue's consumer side.
30
+ *
31
+ * `stop` must **stop accepting new jobs immediately** and only then wait for in-flight handlers. An
32
+ * implementation that drains first would accept a job it has no intention of finishing.
33
+ */
34
+ export interface JobConsumer {
35
+ start(handler: (job: RunJob) => Promise<void>): Promise<void> | void;
36
+ stop(graceMs: number): Promise<void>;
37
+ }
38
+ export type WorkerConfig = {
39
+ /** Jobs handled at once. Above the store's connection pool this queues on the pool instead. */
40
+ readonly concurrency: number;
41
+ /**
42
+ * How long a claim survives without a keepalive. The worker heartbeats at a third of this, so a
43
+ * single slow round trip does not drop a live claim.
44
+ */
45
+ readonly leaseMs: number;
46
+ /** How often to sweep for runs whose lease expired. */
47
+ readonly reapEveryMs: number;
48
+ /** Candidates per sweep. Bounded so a large backlog cannot turn one sweep into a long transaction. */
49
+ readonly reapLimit: number;
50
+ /**
51
+ * How long shutdown waits for in-flight runs.
52
+ *
53
+ * Deliberately shorter than a run's `maxWallClockMs` can be: a long run will be **checkpointed**
54
+ * rather than finished, which is correct — the alternative is a deploy that never completes. See the
55
+ * open question on #107.
56
+ */
57
+ readonly shutdownGraceMs: number;
58
+ };
59
+ export declare const DEFAULT_WORKER_CONFIG: WorkerConfig;
60
+ export type WorkerRuntimeDeps = {
61
+ readonly worker: DurableWorker;
62
+ readonly consumer: JobConsumer;
63
+ /** Reaped runs are re-enqueued rather than executed inline, so recovery goes through the same path. */
64
+ readonly dispatcher: JobDispatcher;
65
+ readonly config?: Partial<WorkerConfig>;
66
+ readonly log?: (message: string, detail?: Readonly<Record<string, unknown>>) => void;
67
+ /** Observability hook; also what the tests assert on. */
68
+ readonly onOutcome?: (job: RunJob, result: ProcessResult) => void;
69
+ readonly onError?: (job: RunJob, error: unknown) => void;
70
+ /** Injectable so a test can drive the reaper without waiting on wall clock. */
71
+ readonly setInterval?: (fn: () => void, ms: number) => {
72
+ readonly clear: () => void;
73
+ };
74
+ };
75
+ export type WorkerStatus = {
76
+ readonly running: boolean;
77
+ readonly inFlight: number;
78
+ readonly processed: number;
79
+ readonly reaped: number;
80
+ };
81
+ export type WorkerRuntime = {
82
+ start(): Promise<void>;
83
+ /** Resolves once consumption has stopped and in-flight work has finished or the grace elapsed. */
84
+ shutdown(reason: string): Promise<{
85
+ readonly graceful: boolean;
86
+ readonly exitCode: number;
87
+ }>;
88
+ status(): WorkerStatus;
89
+ /** Runs one reap sweep now. Exposed so a host can trigger recovery on demand. */
90
+ reapOnce(): Promise<readonly Run[]>;
91
+ };
92
+ export declare const createWorkerRuntime: (deps: WorkerRuntimeDeps) => WorkerRuntime;
93
+ /**
94
+ * Attach OS signal handlers. Separate from `createWorkerRuntime` on purpose: a library that binds to
95
+ * `process` merely by being constructed is hostile to embed and to test, and every test here would
96
+ * otherwise need real signals.
97
+ */
98
+ export declare const installSignalHandlers: (runtime: WorkerRuntime, options?: {
99
+ readonly signals?: readonly NodeJS.Signals[];
100
+ readonly onExit?: (code: number) => void;
101
+ readonly process?: Pick<NodeJS.Process, "on">;
102
+ }) => (() => void);
103
+ //# sourceMappingURL=main.d.ts.map