@sixb/core 0.1.16 → 0.1.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (563) hide show
  1. package/dist/actions/index.js +25 -25
  2. package/dist/actions/types/context.d.ts +2 -2
  3. package/dist/actions/types/context.d.ts.map +1 -1
  4. package/dist/actions/worker.js +1 -1
  5. package/dist/agents/index.js +41 -40
  6. package/dist/agents/index.js.map +1 -1
  7. package/dist/auth/index.js +5 -5
  8. package/dist/authorization/index.js +2 -2
  9. package/dist/blob-storage/download-urls.d.ts +41 -0
  10. package/dist/blob-storage/download-urls.d.ts.map +1 -0
  11. package/dist/blob-storage/execution.d.ts +9 -1
  12. package/dist/blob-storage/execution.d.ts.map +1 -1
  13. package/dist/blob-storage/index.js +5 -3
  14. package/dist/blob-storage/index.js.map +1 -1
  15. package/dist/blob-storage/internal.d.ts +2 -0
  16. package/dist/blob-storage/internal.d.ts.map +1 -0
  17. package/dist/blob-storage/internal.js +11 -0
  18. package/dist/blob-storage/internal.js.map +9 -0
  19. package/dist/bootstrap/index.js +45 -44
  20. package/dist/bootstrap/index.js.map +1 -1
  21. package/dist/{chunk-8x74221e.js → chunk-10styjgk.js} +18 -13
  22. package/dist/{chunk-8x74221e.js.map → chunk-10styjgk.js.map} +3 -3
  23. package/dist/{chunk-hgqpcmyr.js → chunk-1hqwggsn.js} +47 -19
  24. package/dist/chunk-1hqwggsn.js.map +12 -0
  25. package/dist/{chunk-zb47r1c8.js → chunk-23xs4qp9.js} +2 -2
  26. package/dist/{chunk-gmdvns8f.js → chunk-2am3db31.js} +2 -2
  27. package/dist/{chunk-fj1dvp37.js → chunk-2vhg6ysa.js} +1 -1
  28. package/dist/{chunk-dp7yk20b.js → chunk-33saed7g.js} +1 -1
  29. package/dist/{chunk-mdnw7nj8.js → chunk-3rrezgyc.js} +33 -8
  30. package/dist/chunk-3rrezgyc.js.map +15 -0
  31. package/dist/{chunk-jgwb25gc.js → chunk-4as3wt1c.js} +13 -5
  32. package/dist/chunk-4as3wt1c.js.map +12 -0
  33. package/dist/chunk-4ra628t7.js +88 -0
  34. package/dist/chunk-4ra628t7.js.map +10 -0
  35. package/dist/{chunk-chdg6eqq.js → chunk-52c5zb0z.js} +2 -2
  36. package/dist/{chunk-69kt9q43.js → chunk-5be2ath4.js} +1 -1
  37. package/dist/{chunk-qnd7ehe5.js → chunk-5thzcb8g.js} +2 -2
  38. package/dist/{chunk-6z1q7j19.js → chunk-5yd8cchh.js} +1126 -780
  39. package/dist/chunk-5yd8cchh.js.map +66 -0
  40. package/dist/{chunk-3q7z9hd4.js → chunk-6wbm6ymz.js} +234 -40
  41. package/dist/{chunk-3q7z9hd4.js.map → chunk-6wbm6ymz.js.map} +8 -6
  42. package/dist/{chunk-kvysp909.js → chunk-7ecp4v7a.js} +26 -19
  43. package/dist/chunk-7ecp4v7a.js.map +17 -0
  44. package/dist/{chunk-f85hpqnm.js → chunk-99nd4rc1.js} +3 -3
  45. package/dist/{chunk-f85hpqnm.js.map → chunk-99nd4rc1.js.map} +3 -3
  46. package/dist/{chunk-dsv8kdgk.js → chunk-9g6jz9kk.js} +5 -5
  47. package/dist/{chunk-39tqv9hz.js → chunk-9se093qz.js} +5 -6
  48. package/dist/{chunk-39tqv9hz.js.map → chunk-9se093qz.js.map} +3 -4
  49. package/dist/{chunk-7g2gjvah.js → chunk-a7er86jd.js} +1 -1
  50. package/dist/{chunk-zdkcdshk.js → chunk-aq39q04r.js} +41 -3
  51. package/dist/{chunk-zdkcdshk.js.map → chunk-aq39q04r.js.map} +4 -3
  52. package/dist/chunk-b9affek3.js +261 -0
  53. package/dist/chunk-b9affek3.js.map +12 -0
  54. package/dist/{chunk-b9y8hcm2.js → chunk-c2rga5cx.js} +1 -1
  55. package/dist/{chunk-f72rbvsb.js → chunk-cnb5kv05.js} +2 -2
  56. package/dist/chunk-d62kjtbg.js.map +2 -2
  57. package/dist/{chunk-vjqt7q3a.js → chunk-dam456ve.js} +233 -34
  58. package/dist/{chunk-vjqt7q3a.js.map → chunk-dam456ve.js.map} +7 -5
  59. package/dist/{chunk-sbcm1b92.js → chunk-e67ynxcr.js} +3 -3
  60. package/dist/{chunk-xnx96gfy.js → chunk-ere85b8y.js} +17 -8
  61. package/dist/chunk-ere85b8y.js.map +15 -0
  62. package/dist/{chunk-k4kkp58d.js → chunk-ewqa58h9.js} +12 -2
  63. package/dist/chunk-ewqa58h9.js.map +11 -0
  64. package/dist/{chunk-cq44v1a6.js → chunk-f1xh7a32.js} +3 -3
  65. package/dist/{chunk-h4138ww2.js → chunk-fggq3d9k.js} +450 -799
  66. package/dist/chunk-fggq3d9k.js.map +49 -0
  67. package/dist/{chunk-kttr5ftm.js → chunk-gf7yht60.js} +3 -3
  68. package/dist/chunk-h8teetr0.js +9 -0
  69. package/dist/chunk-h8teetr0.js.map +10 -0
  70. package/dist/chunk-hn2r7z41.js.map +1 -1
  71. package/dist/{chunk-nj0020ay.js → chunk-jjrn5v0p.js} +1 -1
  72. package/dist/{chunk-nvymebc1.js → chunk-jnfxmr3f.js} +1 -1
  73. package/dist/{chunk-nkbev3tr.js → chunk-k4g51s47.js} +3 -3
  74. package/dist/{chunk-a0073121.js → chunk-kpx7jrmn.js} +3 -1
  75. package/dist/chunk-kpx7jrmn.js.map +12 -0
  76. package/dist/{chunk-h044jb97.js → chunk-pja9jfz6.js} +172 -10
  77. package/dist/chunk-pja9jfz6.js.map +19 -0
  78. package/dist/chunk-pqajwytd.js.map +1 -1
  79. package/dist/{chunk-mr60an99.js → chunk-qczk96d9.js} +22 -2
  80. package/dist/{chunk-mr60an99.js.map → chunk-qczk96d9.js.map} +3 -3
  81. package/dist/{chunk-s3csryxe.js → chunk-rj69qcrd.js} +8 -2
  82. package/dist/chunk-rj69qcrd.js.map +11 -0
  83. package/dist/{chunk-swxzncrv.js → chunk-rpxz486p.js} +32 -5
  84. package/dist/{chunk-swxzncrv.js.map → chunk-rpxz486p.js.map} +4 -4
  85. package/dist/{chunk-tkdbge6v.js → chunk-rx7h5fdx.js} +1 -1
  86. package/dist/{chunk-tnvafr7g.js → chunk-s7ftxrdr.js} +105 -7
  87. package/dist/chunk-s7ftxrdr.js.map +22 -0
  88. package/dist/{chunk-h866d223.js → chunk-sk0hcjsz.js} +1 -1
  89. package/dist/{chunk-2xvgy3x5.js → chunk-svabwqdq.js} +2 -2
  90. package/dist/{chunk-667x4drs.js → chunk-t9shqx3w.js} +2 -2
  91. package/dist/{chunk-40qb8wpe.js → chunk-tjjgy3z4.js} +26 -23
  92. package/dist/{chunk-40qb8wpe.js.map → chunk-tjjgy3z4.js.map} +3 -3
  93. package/dist/{chunk-gkxsenvp.js → chunk-vp3z1ckh.js} +31 -7
  94. package/dist/{chunk-gkxsenvp.js.map → chunk-vp3z1ckh.js.map} +5 -4
  95. package/dist/{chunk-cxj8y8t0.js → chunk-vre348sb.js} +2 -2
  96. package/dist/{chunk-vsvrv7n8.js → chunk-w9yycd3t.js} +1 -1
  97. package/dist/{chunk-g4rbm4a6.js → chunk-wt4szfec.js} +379 -331
  98. package/dist/chunk-wt4szfec.js.map +14 -0
  99. package/dist/{chunk-a7ne02hk.js → chunk-xcy5ddha.js} +6 -6
  100. package/dist/{chunk-zvmr2hqk.js → chunk-xyvy73tj.js} +4 -3
  101. package/dist/{chunk-zvmr2hqk.js.map → chunk-xyvy73tj.js.map} +3 -3
  102. package/dist/datasets/write.js +1 -1
  103. package/dist/execution/agent.js +38 -37
  104. package/dist/execution/agent.js.map +1 -1
  105. package/dist/execution/authorized-object-reader.d.ts +3 -1
  106. package/dist/execution/authorized-object-reader.d.ts.map +1 -1
  107. package/dist/execution/primitive.js +36 -35
  108. package/dist/execution/primitive.js.map +2 -2
  109. package/dist/execution/request.js +37 -36
  110. package/dist/execution/request.js.map +2 -2
  111. package/dist/index.d.ts +8 -4
  112. package/dist/index.d.ts.map +1 -1
  113. package/dist/index.js +62 -59
  114. package/dist/index.js.map +1 -1
  115. package/dist/lake-storage/in-memory.d.ts +5 -1
  116. package/dist/lake-storage/in-memory.d.ts.map +1 -1
  117. package/dist/lake-storage/index.d.ts +1 -1
  118. package/dist/lake-storage/index.d.ts.map +1 -1
  119. package/dist/lake-storage/index.js +2 -2
  120. package/dist/lake-storage/types.d.ts +26 -0
  121. package/dist/lake-storage/types.d.ts.map +1 -1
  122. package/dist/maintenance/ontology-maintenance.d.ts +5 -1
  123. package/dist/maintenance/ontology-maintenance.d.ts.map +1 -1
  124. package/dist/maintenance/types.d.ts +1 -4
  125. package/dist/maintenance/types.d.ts.map +1 -1
  126. package/dist/materialization/event-envelopes.d.ts +20 -0
  127. package/dist/materialization/event-envelopes.d.ts.map +1 -0
  128. package/dist/materialization/events.d.ts +29 -3
  129. package/dist/materialization/events.d.ts.map +1 -1
  130. package/dist/materialization/index.d.ts +4 -1
  131. package/dist/materialization/index.d.ts.map +1 -1
  132. package/dist/materialization/index.js +9 -3
  133. package/dist/materialization/index.js.map +1 -1
  134. package/dist/materializer/edits/plan.d.ts +2 -10
  135. package/dist/materializer/edits/plan.d.ts.map +1 -1
  136. package/dist/materializer/effective/build-events.d.ts +5 -21
  137. package/dist/materializer/effective/build-events.d.ts.map +1 -1
  138. package/dist/materializer/effective/vector-indexing.d.ts +2 -3
  139. package/dist/materializer/effective/vector-indexing.d.ts.map +1 -1
  140. package/dist/materializer/effective/vectors.d.ts +2 -3
  141. package/dist/materializer/effective/vectors.d.ts.map +1 -1
  142. package/dist/materializer/execution/scope.d.ts +1 -11
  143. package/dist/materializer/execution/scope.d.ts.map +1 -1
  144. package/dist/materializer/execution/work-executor.d.ts +5 -7
  145. package/dist/materializer/execution/work-executor.d.ts.map +1 -1
  146. package/dist/materializer/execution/work-records.d.ts +1 -3
  147. package/dist/materializer/execution/work-records.d.ts.map +1 -1
  148. package/dist/materializer/index.js +31 -32
  149. package/dist/materializer/index.js.map +1 -1
  150. package/dist/materializer/projections/replace.d.ts.map +1 -1
  151. package/dist/materializer/projections/replacement-plan.d.ts +8 -11
  152. package/dist/materializer/projections/replacement-plan.d.ts.map +1 -1
  153. package/dist/materializer/shared/batching.d.ts +5 -0
  154. package/dist/materializer/shared/batching.d.ts.map +1 -1
  155. package/dist/materializer/telemetry/plan.d.ts +2 -8
  156. package/dist/materializer/telemetry/plan.d.ts.map +1 -1
  157. package/dist/models/audio/catalog.d.ts +20 -0
  158. package/dist/models/audio/catalog.d.ts.map +1 -0
  159. package/dist/models/audio/errors.d.ts +8 -0
  160. package/dist/models/audio/errors.d.ts.map +1 -0
  161. package/dist/models/audio/file.d.ts +9 -0
  162. package/dist/models/audio/file.d.ts.map +1 -0
  163. package/dist/models/audio/runtime.d.ts +8 -0
  164. package/dist/models/audio/runtime.d.ts.map +1 -0
  165. package/dist/models/audio/types.d.ts +60 -0
  166. package/dist/models/audio/types.d.ts.map +1 -0
  167. package/dist/models/catalog.d.ts +23 -1
  168. package/dist/models/catalog.d.ts.map +1 -1
  169. package/dist/models/definitions.d.ts +1 -1
  170. package/dist/models/definitions.d.ts.map +1 -1
  171. package/dist/models/events.d.ts +3 -0
  172. package/dist/models/events.d.ts.map +1 -1
  173. package/dist/models/execution/index.js +22 -22
  174. package/dist/models/execution/model-call-admission.d.ts +6 -1
  175. package/dist/models/execution/model-call-admission.d.ts.map +1 -1
  176. package/dist/models/execution/model-call-limits.d.ts.map +1 -1
  177. package/dist/models/execution/model-call-recorder.d.ts.map +1 -1
  178. package/dist/models/execution/reranking.d.ts +4 -0
  179. package/dist/models/execution/reranking.d.ts.map +1 -0
  180. package/dist/models/execution/usage.d.ts.map +1 -1
  181. package/dist/models/execution.d.ts +2 -1
  182. package/dist/models/execution.d.ts.map +1 -1
  183. package/dist/models/generation-types.d.ts +2 -0
  184. package/dist/models/generation-types.d.ts.map +1 -1
  185. package/dist/models/index.d.ts +7 -1
  186. package/dist/models/index.d.ts.map +1 -1
  187. package/dist/models/index.js +6 -2
  188. package/dist/models/index.js.map +1 -1
  189. package/dist/models/pricing.d.ts +2 -2
  190. package/dist/models/pricing.d.ts.map +1 -1
  191. package/dist/models/reranking-model.d.ts +49 -0
  192. package/dist/models/reranking-model.d.ts.map +1 -0
  193. package/dist/objects/index.js +25 -25
  194. package/dist/objects/query/executor.d.ts +2 -1
  195. package/dist/objects/query/executor.d.ts.map +1 -1
  196. package/dist/objects/query/index.d.ts +1 -1
  197. package/dist/objects/query/index.d.ts.map +1 -1
  198. package/dist/objects/query/index.js +22 -22
  199. package/dist/objects/query/ir.d.ts +8 -1
  200. package/dist/objects/query/ir.d.ts.map +1 -1
  201. package/dist/objects/query/normalize.d.ts.map +1 -1
  202. package/dist/objects/query/planner.d.ts +1 -1
  203. package/dist/objects/query/planner.d.ts.map +1 -1
  204. package/dist/objects/query/primary-id.d.ts +7 -0
  205. package/dist/objects/query/primary-id.d.ts.map +1 -0
  206. package/dist/objects/query/reranking-executor.d.ts +13 -0
  207. package/dist/objects/query/reranking-executor.d.ts.map +1 -0
  208. package/dist/objects/query/reranking-limits.d.ts +3 -0
  209. package/dist/objects/query/reranking-limits.d.ts.map +1 -0
  210. package/dist/objects/query/reranking-query.d.ts +4 -0
  211. package/dist/objects/query/reranking-query.d.ts.map +1 -0
  212. package/dist/objects/query/validate.d.ts.map +1 -1
  213. package/dist/objects/sdk/query-builder.d.ts +8 -0
  214. package/dist/objects/sdk/query-builder.d.ts.map +1 -1
  215. package/dist/objects/sdk/query-executor.d.ts +2 -2
  216. package/dist/objects/sdk/query-executor.d.ts.map +1 -1
  217. package/dist/objects/sdk/runtime-query-executor.d.ts.map +1 -1
  218. package/dist/objects/vectors/indexing-dispatch.d.ts +21 -0
  219. package/dist/objects/vectors/indexing-dispatch.d.ts.map +1 -0
  220. package/dist/objects/vectors/query.d.ts.map +1 -1
  221. package/dist/ontology/builders.d.ts.map +1 -1
  222. package/dist/ontology/index.d.ts +1 -1
  223. package/dist/ontology/index.d.ts.map +1 -1
  224. package/dist/ontology/index.js +3 -3
  225. package/dist/ontology/internal.js +2 -2
  226. package/dist/ontology/query-capabilities.d.ts +1 -0
  227. package/dist/ontology/query-capabilities.d.ts.map +1 -1
  228. package/dist/ontology/types.d.ts +20 -0
  229. package/dist/ontology/types.d.ts.map +1 -1
  230. package/dist/ontology/validation/query.d.ts.map +1 -1
  231. package/dist/pipelines/index.js +23 -23
  232. package/dist/projections/internal.js +22 -22
  233. package/dist/query.d.ts +4 -2
  234. package/dist/query.d.ts.map +1 -1
  235. package/dist/query.js +2 -2
  236. package/dist/queues/in-memory.d.ts +3 -0
  237. package/dist/queues/in-memory.d.ts.map +1 -1
  238. package/dist/queues/index.js +1 -1
  239. package/dist/queues/types.d.ts +4 -0
  240. package/dist/queues/types.d.ts.map +1 -1
  241. package/dist/runtime/host.d.ts +1 -0
  242. package/dist/runtime/host.d.ts.map +1 -1
  243. package/dist/runtime/index.d.ts +1 -1
  244. package/dist/runtime/index.d.ts.map +1 -1
  245. package/dist/runtime/internal.d.ts +1 -0
  246. package/dist/runtime/internal.d.ts.map +1 -1
  247. package/dist/runtime/internal.js +28 -26
  248. package/dist/runtime/internal.js.map +1 -1
  249. package/dist/runtime/public-origin.d.ts +10 -0
  250. package/dist/runtime/public-origin.d.ts.map +1 -0
  251. package/dist/runtime/sixb.d.ts +1 -0
  252. package/dist/runtime/sixb.d.ts.map +1 -1
  253. package/dist/runtime/types.d.ts +12 -2
  254. package/dist/runtime/types.d.ts.map +1 -1
  255. package/dist/sandboxes/index.js +5 -5
  256. package/dist/shares/internal.js +19 -19
  257. package/dist/storage/agents/provider.js +2 -2
  258. package/dist/storage/ai-cost/analytics.d.ts +1 -0
  259. package/dist/storage/ai-cost/analytics.d.ts.map +1 -1
  260. package/dist/storage/ai-cost/provider.js +3 -3
  261. package/dist/storage/ai-cost/types.d.ts +1 -1
  262. package/dist/storage/ai-cost/types.d.ts.map +1 -1
  263. package/dist/storage/ai-cost/validation.d.ts.map +1 -1
  264. package/dist/storage/ai-limits/enforcement.js +2 -2
  265. package/dist/storage/ai-limits/in-memory.d.ts.map +1 -1
  266. package/dist/storage/ai-limits/provider.d.ts +7 -2
  267. package/dist/storage/ai-limits/provider.d.ts.map +1 -1
  268. package/dist/storage/ai-limits/provider.js +4 -2
  269. package/dist/storage/ai-limits/provider.js.map +1 -1
  270. package/dist/storage/ai-limits/types.d.ts +3 -0
  271. package/dist/storage/ai-limits/types.d.ts.map +1 -1
  272. package/dist/storage/ai-usage/record.d.ts.map +1 -1
  273. package/dist/storage/ai-usage/types.d.ts +8 -4
  274. package/dist/storage/ai-usage/types.d.ts.map +1 -1
  275. package/dist/storage/ai-usage/usage.d.ts.map +1 -1
  276. package/dist/storage/file-download-grants/in-memory.d.ts +11 -0
  277. package/dist/storage/file-download-grants/in-memory.d.ts.map +1 -0
  278. package/dist/storage/file-download-grants/index.d.ts +3 -0
  279. package/dist/storage/file-download-grants/index.d.ts.map +1 -0
  280. package/dist/storage/file-download-grants/record.d.ts +11 -0
  281. package/dist/storage/file-download-grants/record.d.ts.map +1 -0
  282. package/dist/storage/file-download-grants/types.d.ts +44 -0
  283. package/dist/storage/file-download-grants/types.d.ts.map +1 -0
  284. package/dist/storage/file-upload-sessions/provider.js +6 -6
  285. package/dist/storage/in-memory/index.d.ts +4 -0
  286. package/dist/storage/in-memory/index.d.ts.map +1 -1
  287. package/dist/storage/index.d.ts +5 -3
  288. package/dist/storage/index.d.ts.map +1 -1
  289. package/dist/storage/index.js +30 -30
  290. package/dist/storage/index.js.map +1 -1
  291. package/dist/storage/internal.d.ts +1 -0
  292. package/dist/storage/internal.d.ts.map +1 -1
  293. package/dist/storage/internal.js +8 -4
  294. package/dist/storage/internal.js.map +1 -1
  295. package/dist/storage/migrations/index.d.ts +1 -1
  296. package/dist/storage/migrations/index.d.ts.map +1 -1
  297. package/dist/storage/migrations/migrate-storage.d.ts +3 -3
  298. package/dist/storage/migrations/migrate-storage.d.ts.map +1 -1
  299. package/dist/storage/migrations/types.d.ts +7 -0
  300. package/dist/storage/migrations/types.d.ts.map +1 -1
  301. package/dist/storage/objects/in-memory/query.d.ts.map +1 -1
  302. package/dist/storage/objects/in-memory/selected-reader.d.ts.map +1 -1
  303. package/dist/storage/objects/in-memory/storage.d.ts.map +1 -1
  304. package/dist/storage/objects/query-preparation.d.ts +39 -0
  305. package/dist/storage/objects/query-preparation.d.ts.map +1 -0
  306. package/dist/storage/objects/types.d.ts +2 -0
  307. package/dist/storage/objects/types.d.ts.map +1 -1
  308. package/dist/storage/ontology/commits.d.ts +1 -6
  309. package/dist/storage/ontology/commits.d.ts.map +1 -1
  310. package/dist/storage/ontology/in-memory/index.d.ts +4 -2
  311. package/dist/storage/ontology/in-memory/index.d.ts.map +1 -1
  312. package/dist/storage/ontology/in-memory/materializations-state.d.ts +0 -2
  313. package/dist/storage/ontology/in-memory/materializations-state.d.ts.map +1 -1
  314. package/dist/storage/ontology/in-memory/materializations-work.d.ts +1 -3
  315. package/dist/storage/ontology/in-memory/materializations-work.d.ts.map +1 -1
  316. package/dist/storage/ontology/in-memory/materializations.d.ts +23 -57
  317. package/dist/storage/ontology/in-memory/materializations.d.ts.map +1 -1
  318. package/dist/storage/ontology/in-memory/outbox.d.ts +3 -2
  319. package/dist/storage/ontology/in-memory/outbox.d.ts.map +1 -1
  320. package/dist/storage/ontology/in-memory/replacement-plans.d.ts +104 -0
  321. package/dist/storage/ontology/in-memory/replacement-plans.d.ts.map +1 -0
  322. package/dist/storage/ontology/in-memory/shared-state.d.ts +17 -2
  323. package/dist/storage/ontology/in-memory/shared-state.d.ts.map +1 -1
  324. package/dist/storage/ontology/in-memory/source-roots.d.ts +3 -1
  325. package/dist/storage/ontology/in-memory/source-roots.d.ts.map +1 -1
  326. package/dist/storage/ontology/in-memory/sources.d.ts +1 -2
  327. package/dist/storage/ontology/in-memory/sources.d.ts.map +1 -1
  328. package/dist/storage/ontology/in-memory/state-reader.d.ts +25 -0
  329. package/dist/storage/ontology/in-memory/state-reader.d.ts.map +1 -0
  330. package/dist/storage/ontology/index.d.ts +6 -3
  331. package/dist/storage/ontology/index.d.ts.map +1 -1
  332. package/dist/storage/ontology/materializations.d.ts +40 -59
  333. package/dist/storage/ontology/materializations.d.ts.map +1 -1
  334. package/dist/storage/ontology/outbox.d.ts +4 -6
  335. package/dist/storage/ontology/outbox.d.ts.map +1 -1
  336. package/dist/storage/ontology/provider-work.d.ts +114 -3
  337. package/dist/storage/ontology/provider-work.d.ts.map +1 -1
  338. package/dist/storage/ontology/provider.d.ts +14 -39
  339. package/dist/storage/ontology/provider.d.ts.map +1 -1
  340. package/dist/storage/ontology/provider.js +26 -12
  341. package/dist/storage/ontology/provider.js.map +1 -1
  342. package/dist/storage/ontology/replacement-plans.d.ts +103 -0
  343. package/dist/storage/ontology/replacement-plans.d.ts.map +1 -0
  344. package/dist/storage/ontology/sources.d.ts +0 -8
  345. package/dist/storage/ontology/sources.d.ts.map +1 -1
  346. package/dist/storage/operation-scope.d.ts.map +1 -1
  347. package/dist/storage/operation-scope.js +1 -1
  348. package/dist/storage/projection-runs/in-memory.d.ts.map +1 -1
  349. package/dist/storage/projection-runs/types.d.ts +2 -0
  350. package/dist/storage/projection-runs/types.d.ts.map +1 -1
  351. package/dist/storage/types.d.ts +2 -0
  352. package/dist/storage/types.d.ts.map +1 -1
  353. package/dist/storage/workflow-runs/provider.js +6 -6
  354. package/dist/syncs/index.js +24 -24
  355. package/dist/testing/ai-cost-storage-contract.d.ts.map +1 -1
  356. package/dist/testing/ai-limit-storage-contract.d.ts.map +1 -1
  357. package/dist/testing/ai-usage-storage-contract.d.ts.map +1 -1
  358. package/dist/testing/file-download-grant-storage-contract.d.ts +9 -0
  359. package/dist/testing/file-download-grant-storage-contract.d.ts.map +1 -0
  360. package/dist/testing/index.d.ts +1 -0
  361. package/dist/testing/index.d.ts.map +1 -1
  362. package/dist/testing/index.js +2549 -1490
  363. package/dist/testing/index.js.map +14 -13
  364. package/dist/testing/lake-storage-contract.d.ts.map +1 -1
  365. package/dist/testing/materializer-storage-contract.d.ts +14 -0
  366. package/dist/testing/materializer-storage-contract.d.ts.map +1 -1
  367. package/dist/testing/object-read-scope-contract.d.ts.map +1 -1
  368. package/dist/testing/ontology-contract-fixture.d.ts +5 -1
  369. package/dist/testing/ontology-contract-fixture.d.ts.map +1 -1
  370. package/dist/testing/ontology-storage-contract.d.ts.map +1 -1
  371. package/dist/testing/projection-run-storage-contract.d.ts.map +1 -1
  372. package/dist/testing/queues-contract.d.ts.map +1 -1
  373. package/dist/webhooks/internal.js +20 -20
  374. package/dist/workers/index.js +15 -17
  375. package/dist/workers/index.js.map +3 -3
  376. package/dist/workers/queue-worker.d.ts +3 -2
  377. package/dist/workers/queue-worker.d.ts.map +1 -1
  378. package/dist/workflows/index.js +42 -41
  379. package/dist/workflows/index.js.map +1 -1
  380. package/package.json +7 -1
  381. package/src/actions/types/context.ts +5 -2
  382. package/src/blob-storage/download-urls.ts +138 -0
  383. package/src/blob-storage/execution.ts +32 -1
  384. package/src/blob-storage/internal.ts +1 -0
  385. package/src/execution/authorized-object-reader.ts +8 -1
  386. package/src/index.ts +25 -0
  387. package/src/lake-storage/in-memory.ts +38 -9
  388. package/src/lake-storage/index.ts +2 -0
  389. package/src/lake-storage/types.ts +30 -0
  390. package/src/maintenance/ontology-maintenance.ts +50 -23
  391. package/src/maintenance/types.ts +1 -4
  392. package/src/materialization/event-envelopes.ts +168 -0
  393. package/src/materialization/events.ts +40 -4
  394. package/src/materialization/index.ts +9 -1
  395. package/src/materializer/README.md +55 -10
  396. package/src/materializer/edits/commit.ts +3 -19
  397. package/src/materializer/edits/plan.ts +20 -71
  398. package/src/materializer/effective/build-events.ts +42 -136
  399. package/src/materializer/effective/vector-indexing.ts +2 -3
  400. package/src/materializer/effective/vectors.ts +2 -3
  401. package/src/materializer/execution/scope.ts +1 -38
  402. package/src/materializer/execution/work-executor.ts +21 -115
  403. package/src/materializer/execution/work-records.ts +0 -15
  404. package/src/materializer/projections/replace.ts +106 -63
  405. package/src/materializer/projections/replacement-plan.ts +73 -218
  406. package/src/materializer/shared/batching.ts +7 -0
  407. package/src/materializer/telemetry/append.ts +2 -14
  408. package/src/materializer/telemetry/plan.ts +6 -43
  409. package/src/models/audio/catalog.ts +90 -0
  410. package/src/models/audio/errors.ts +19 -0
  411. package/src/models/audio/file.ts +78 -0
  412. package/src/models/audio/runtime.ts +182 -0
  413. package/src/models/audio/types.ts +69 -0
  414. package/src/models/catalog.ts +52 -5
  415. package/src/models/definitions.ts +8 -1
  416. package/src/models/events.ts +3 -0
  417. package/src/models/execution/README.md +20 -1
  418. package/src/models/execution/model-call-admission.ts +6 -1
  419. package/src/models/execution/model-call-limits.ts +21 -11
  420. package/src/models/execution/model-call-recorder.ts +1 -0
  421. package/src/models/execution/model-call-recovery.ts +1 -0
  422. package/src/models/execution/reranking.ts +137 -0
  423. package/src/models/execution/usage.ts +1 -0
  424. package/src/models/execution.ts +5 -1
  425. package/src/models/generation-types.ts +2 -0
  426. package/src/models/index.ts +32 -0
  427. package/src/models/pricing.ts +2 -1
  428. package/src/models/reranking-model.ts +114 -0
  429. package/src/objects/query/executor.ts +42 -2
  430. package/src/objects/query/explain.ts +8 -0
  431. package/src/objects/query/index.ts +1 -0
  432. package/src/objects/query/ir.ts +10 -0
  433. package/src/objects/query/normalize.ts +2 -0
  434. package/src/objects/query/planner.ts +12 -1
  435. package/src/objects/query/primary-id.ts +56 -0
  436. package/src/objects/query/reranking-executor.ts +91 -0
  437. package/src/objects/query/reranking-limits.ts +3 -0
  438. package/src/objects/query/reranking-query.ts +8 -0
  439. package/src/objects/query/validate.ts +33 -1
  440. package/src/objects/sdk/query-builder.ts +15 -2
  441. package/src/objects/sdk/query-executor.ts +2 -2
  442. package/src/objects/sdk/runtime-query-executor.ts +7 -4
  443. package/src/objects/vectors/README.md +6 -2
  444. package/src/objects/vectors/indexing-dispatch.ts +83 -0
  445. package/src/objects/vectors/query.ts +3 -0
  446. package/src/ontology/builders.ts +1 -0
  447. package/src/ontology/index.ts +2 -0
  448. package/src/ontology/query-capabilities.ts +1 -1
  449. package/src/ontology/types.ts +21 -0
  450. package/src/ontology/validation/query.ts +57 -0
  451. package/src/query.ts +4 -0
  452. package/src/queues/in-memory.ts +54 -19
  453. package/src/queues/types.ts +4 -0
  454. package/src/runtime/host.ts +28 -1
  455. package/src/runtime/index.ts +1 -0
  456. package/src/runtime/internal.ts +1 -0
  457. package/src/runtime/public-origin.ts +33 -0
  458. package/src/runtime/sixb.ts +13 -2
  459. package/src/runtime/types.ts +12 -2
  460. package/src/storage/ai-cost/analytics.ts +2 -0
  461. package/src/storage/ai-cost/codec.ts +2 -1
  462. package/src/storage/ai-cost/types.ts +1 -0
  463. package/src/storage/ai-cost/validation.ts +21 -8
  464. package/src/storage/ai-limits/in-memory.ts +10 -2
  465. package/src/storage/ai-limits/provider.ts +29 -2
  466. package/src/storage/ai-limits/types.ts +3 -0
  467. package/src/storage/ai-usage/record.ts +9 -0
  468. package/src/storage/ai-usage/types.ts +8 -4
  469. package/src/storage/ai-usage/usage.ts +2 -0
  470. package/src/storage/file-download-grants/in-memory.ts +73 -0
  471. package/src/storage/file-download-grants/index.ts +8 -0
  472. package/src/storage/file-download-grants/record.ts +47 -0
  473. package/src/storage/file-download-grants/types.ts +48 -0
  474. package/src/storage/in-memory/index.ts +13 -2
  475. package/src/storage/index.ts +31 -13
  476. package/src/storage/internal.ts +1 -0
  477. package/src/storage/migrations/index.ts +1 -0
  478. package/src/storage/migrations/migrate-storage.ts +36 -12
  479. package/src/storage/migrations/types.ts +8 -0
  480. package/src/storage/objects/in-memory/query.ts +2 -0
  481. package/src/storage/objects/in-memory/selected-reader.ts +1 -0
  482. package/src/storage/objects/in-memory/storage.ts +1 -0
  483. package/src/storage/objects/query-preparation.ts +77 -0
  484. package/src/storage/objects/types.ts +2 -0
  485. package/src/storage/ontology/commits.ts +1 -6
  486. package/src/storage/ontology/in-memory/index.ts +28 -9
  487. package/src/storage/ontology/in-memory/materializations-finalization.ts +35 -68
  488. package/src/storage/ontology/in-memory/materializations-state.ts +0 -22
  489. package/src/storage/ontology/in-memory/materializations-work.ts +0 -45
  490. package/src/storage/ontology/in-memory/materializations.ts +172 -653
  491. package/src/storage/ontology/in-memory/outbox.ts +37 -29
  492. package/src/storage/ontology/in-memory/replacement-plans.ts +697 -0
  493. package/src/storage/ontology/in-memory/shared-state.ts +24 -2
  494. package/src/storage/ontology/in-memory/source-roots.ts +7 -2
  495. package/src/storage/ontology/in-memory/sources.ts +7 -33
  496. package/src/storage/ontology/in-memory/state-reader.ts +245 -0
  497. package/src/storage/ontology/index.ts +19 -13
  498. package/src/storage/ontology/materializations.ts +45 -74
  499. package/src/storage/ontology/outbox.ts +4 -7
  500. package/src/storage/ontology/provider-header-validation.ts +0 -57
  501. package/src/storage/ontology/provider-work.ts +381 -90
  502. package/src/storage/ontology/provider.ts +74 -171
  503. package/src/storage/ontology/replacement-plans.ts +117 -0
  504. package/src/storage/ontology/sources.ts +0 -12
  505. package/src/storage/operation-scope.ts +33 -1
  506. package/src/storage/projection-runs/in-memory.ts +7 -1
  507. package/src/storage/projection-runs/types.ts +2 -0
  508. package/src/storage/types.ts +2 -0
  509. package/src/testing/ai-cost-storage-contract.ts +1 -0
  510. package/src/testing/ai-limit-storage-contract.ts +62 -0
  511. package/src/testing/ai-usage-storage-contract.ts +31 -0
  512. package/src/testing/file-download-grant-storage-contract.ts +139 -0
  513. package/src/testing/index.ts +4 -0
  514. package/src/testing/lake-storage-contract.ts +87 -1
  515. package/src/testing/materializer-storage-contract.ts +1008 -1
  516. package/src/testing/object-read-scope-contract.ts +1 -0
  517. package/src/testing/ontology-contract-fixture.ts +11 -87
  518. package/src/testing/ontology-storage-contract.ts +70 -39
  519. package/src/testing/projection-run-storage-contract.ts +14 -0
  520. package/src/testing/queues-contract.ts +59 -0
  521. package/src/workers/queue-worker.ts +19 -16
  522. package/dist/chunk-6z1q7j19.js.map +0 -62
  523. package/dist/chunk-7ytf06d3.js +0 -113
  524. package/dist/chunk-7ytf06d3.js.map +0 -10
  525. package/dist/chunk-a0073121.js.map +0 -12
  526. package/dist/chunk-g4rbm4a6.js.map +0 -15
  527. package/dist/chunk-h044jb97.js.map +0 -15
  528. package/dist/chunk-h4138ww2.js.map +0 -50
  529. package/dist/chunk-hgqpcmyr.js.map +0 -12
  530. package/dist/chunk-jgwb25gc.js.map +0 -12
  531. package/dist/chunk-k4kkp58d.js.map +0 -11
  532. package/dist/chunk-kvysp909.js.map +0 -17
  533. package/dist/chunk-mdnw7nj8.js.map +0 -15
  534. package/dist/chunk-s3csryxe.js.map +0 -11
  535. package/dist/chunk-tnvafr7g.js.map +0 -19
  536. package/dist/chunk-xnx96gfy.js.map +0 -15
  537. package/dist/materializer/execution/plan-stream.d.ts +0 -8
  538. package/dist/materializer/execution/plan-stream.d.ts.map +0 -1
  539. package/src/materializer/execution/plan-stream.ts +0 -121
  540. /package/dist/{chunk-zb47r1c8.js.map → chunk-23xs4qp9.js.map} +0 -0
  541. /package/dist/{chunk-gmdvns8f.js.map → chunk-2am3db31.js.map} +0 -0
  542. /package/dist/{chunk-fj1dvp37.js.map → chunk-2vhg6ysa.js.map} +0 -0
  543. /package/dist/{chunk-dp7yk20b.js.map → chunk-33saed7g.js.map} +0 -0
  544. /package/dist/{chunk-chdg6eqq.js.map → chunk-52c5zb0z.js.map} +0 -0
  545. /package/dist/{chunk-69kt9q43.js.map → chunk-5be2ath4.js.map} +0 -0
  546. /package/dist/{chunk-qnd7ehe5.js.map → chunk-5thzcb8g.js.map} +0 -0
  547. /package/dist/{chunk-dsv8kdgk.js.map → chunk-9g6jz9kk.js.map} +0 -0
  548. /package/dist/{chunk-7g2gjvah.js.map → chunk-a7er86jd.js.map} +0 -0
  549. /package/dist/{chunk-b9y8hcm2.js.map → chunk-c2rga5cx.js.map} +0 -0
  550. /package/dist/{chunk-f72rbvsb.js.map → chunk-cnb5kv05.js.map} +0 -0
  551. /package/dist/{chunk-sbcm1b92.js.map → chunk-e67ynxcr.js.map} +0 -0
  552. /package/dist/{chunk-cq44v1a6.js.map → chunk-f1xh7a32.js.map} +0 -0
  553. /package/dist/{chunk-kttr5ftm.js.map → chunk-gf7yht60.js.map} +0 -0
  554. /package/dist/{chunk-nj0020ay.js.map → chunk-jjrn5v0p.js.map} +0 -0
  555. /package/dist/{chunk-nvymebc1.js.map → chunk-jnfxmr3f.js.map} +0 -0
  556. /package/dist/{chunk-nkbev3tr.js.map → chunk-k4g51s47.js.map} +0 -0
  557. /package/dist/{chunk-tkdbge6v.js.map → chunk-rx7h5fdx.js.map} +0 -0
  558. /package/dist/{chunk-h866d223.js.map → chunk-sk0hcjsz.js.map} +0 -0
  559. /package/dist/{chunk-2xvgy3x5.js.map → chunk-svabwqdq.js.map} +0 -0
  560. /package/dist/{chunk-667x4drs.js.map → chunk-t9shqx3w.js.map} +0 -0
  561. /package/dist/{chunk-cxj8y8t0.js.map → chunk-vre348sb.js.map} +0 -0
  562. /package/dist/{chunk-vsvrv7n8.js.map → chunk-w9yycd3t.js.map} +0 -0
  563. /package/dist/{chunk-a7ne02hk.js.map → chunk-xcy5ddha.js.map} +0 -0
@@ -23,6 +23,8 @@ import type {
23
23
  OntologyMaterializationStorage,
24
24
  OntologySourceRecord,
25
25
  OntologyStorage,
26
+ ReplacementPlanRef,
27
+ ReplacementPlanStatus,
26
28
  } from "../../storage/ontology"
27
29
  import type { MaterializerContext, MaterializerStorage } from "../context"
28
30
  import { replayCommitRecord, withSerializationRetry } from "../execution/commit-lifecycle"
@@ -34,7 +36,7 @@ import {
34
36
  type MaterializerExecution,
35
37
  prepareMaterializerExecution,
36
38
  } from "../execution/scope"
37
- import { drainStagedEvents, drainStagedWork } from "../execution/work-executor"
39
+ import { applyStagedWork } from "../execution/work-executor"
38
40
  import type { MaterializerCommand } from "../materializer"
39
41
  import { throwIfAborted } from "../shared/abort"
40
42
  import {
@@ -51,7 +53,7 @@ import {
51
53
  } from "../shared/normalize"
52
54
  import { createProjectionRunMaterializationIdentity } from "../shared/projection-run"
53
55
  import { createProjectionEntryValidator } from "./entry-validator"
54
- import { planProjectionReplacement } from "./replacement-plan"
56
+ import { type ProjectionReplacementPlanInput, planProjectionReplacement } from "./replacement-plan"
55
57
  import {
56
58
  type StagedProjectionMaterialization,
57
59
  stageProjectionMaterialization,
@@ -324,25 +326,68 @@ async function stageCandidateEntries(
324
326
  return stageProjectionMaterialization(context, { ...input, signal })
325
327
  }
326
328
 
329
+ /**
330
+ * Rounds of planning a commit tolerates before it gives the delivery back: each round plans again
331
+ * only what changed under the previous one, and the next delivery resumes the same plan. A commit
332
+ * whose refresh finds few stale identities plans them in its own transaction instead, so a round
333
+ * ends with a commit unless writes keep changing many of the plan's inputs.
334
+ */
335
+ const MAX_PLAN_ROUNDS = 3
336
+
327
337
  async function commitProjectionCandidate(
328
338
  context: MaterializerContext,
329
339
  command: PreparedProjectionReplacement,
330
340
  ready: ReadyProjectionReplacement
331
341
  ): Promise<ProjectionCommitResult> {
332
- return withSerializationRetry(context, () =>
333
- context.storage.transaction(
334
- (storage) => executeProjectionTransaction(context, storage, command, ready),
335
- { isolation: "serializable" }
342
+ const plans = context.storage.ontology.replacementPlans
343
+ const plan: ReplacementPlanRef = {
344
+ projectId: context.projectId,
345
+ source: command.source,
346
+ materializationId: ready.materializationId,
347
+ execution: command.execution,
348
+ }
349
+ // A resumed plan keeps the commit time it was opened with, since its planned rows and events
350
+ // carry that time; its session begins with the commit at exactly that time.
351
+ const { committedAt } = await plans.open({
352
+ ...plan,
353
+ commit: projectionCommit(context.projectId, command, ready.identity),
354
+ })
355
+ const commit = projectionCommit(context.projectId, command, { ...ready.identity, committedAt })
356
+ for (let round = 1; ; round += 1) {
357
+ await planProjectionReplacement(context, plans, plan, planInput(command, commit))
358
+ const attempt = await withSerializationRetry(context, () =>
359
+ context.storage.transaction(
360
+ (storage) => executeProjectionTransaction(context, storage, command, ready, commit, plan),
361
+ { isolation: "serializable" }
362
+ )
336
363
  )
337
- )
364
+ if (attempt.kind === "committed") return attempt.result
365
+ if (round === MAX_PLAN_ROUNDS) {
366
+ throw new MaterializationConflictError(
367
+ "effective-state",
368
+ `Projection '${command.source.projectionId}' inputs kept changing while it was planned ` +
369
+ `(${attempt.unplanned} changed during the last of ${MAX_PLAN_ROUNDS} rounds).`
370
+ )
371
+ }
372
+ }
338
373
  }
339
374
 
375
+ /**
376
+ * One commit transaction. `stale` commits only the refresh: the identities whose inputs changed
377
+ * since they were planned are unplanned, for the next round to plan again outside the transaction.
378
+ */
379
+ type ProjectionCommitAttempt =
380
+ | { readonly kind: "committed"; readonly result: ProjectionCommitResult }
381
+ | { readonly kind: "stale"; readonly unplanned: number }
382
+
340
383
  async function executeProjectionTransaction(
341
384
  context: MaterializerContext,
342
385
  storage: MaterializerStorage,
343
386
  command: PreparedProjectionReplacement,
344
- ready: ReadyProjectionReplacement
345
- ): Promise<ProjectionCommitResult> {
387
+ ready: ReadyProjectionReplacement,
388
+ commit: OntologyCommitWrite,
389
+ plan: ReplacementPlanRef
390
+ ): Promise<ProjectionCommitAttempt> {
346
391
  await ensureMaterializerExecution(storage.executions, command.scopeExecution)
347
392
  await assertProjectionExecution(storage, context.projectId, command)
348
393
  const replay = await replayCommitRecord(
@@ -351,7 +396,7 @@ async function executeProjectionTransaction(
351
396
  command.scopeExecution.executionId,
352
397
  storage
353
398
  )
354
- if (replay) return projectionReplayResult(replay, command)
399
+ if (replay) return { kind: "committed", result: projectionReplayResult(replay, command) }
355
400
 
356
401
  if (command.base) {
357
402
  const active = await storage.ontology.sources.getActive({
@@ -361,10 +406,11 @@ async function executeProjectionTransaction(
361
406
  validateProjectionWatermark(active, command.datasetVersion)
362
407
  assertDeltaBase(active, command)
363
408
  }
364
- const origin = projectionOrigin(command)
365
409
  throwIfAborted(command.signal)
410
+ const status = await refreshProjectionPlan(context, storage, command, commit, plan)
411
+ if (!status.fresh) return { kind: "stale", unplanned: status.unplanned }
366
412
  const session = await storage.ontology.materializations.begin({
367
- commit: projectionCommit(context.projectId, command, ready.identity, origin),
413
+ commit,
368
414
  expected: {
369
415
  sources: [ready.expectedSource],
370
416
  objects: [],
@@ -372,33 +418,48 @@ async function executeProjectionTransaction(
372
418
  linkScopes: [],
373
419
  points: [],
374
420
  },
421
+ plan: { source: command.source, materializationId: ready.materializationId },
375
422
  })
376
- const counts = await planReadyProjection(
377
- context,
378
- storage.ontology.materializations,
379
- session,
380
- command,
381
- ready,
382
- origin
383
- )
384
- await drainStagedWork(context, storage, session, command.signal, true)
385
- const eventCount = await drainStagedEvents(
386
- context,
387
- storage.ontology.materializations,
388
- session,
389
- ready.identity,
390
- command.signal
391
- )
423
+ const eventCount = await applyStagedWork(context, storage, session, command.signal, true)
392
424
  const result: ProjectionCommitResult = {
393
425
  kind: "projection",
394
- commitId: ready.identity.commitId,
426
+ commitId: commit.id,
395
427
  created: true,
396
428
  eventCount,
397
- committedAt: ready.identity.committedAt,
398
- counts,
429
+ committedAt: commit.committedAt,
430
+ counts: status.counts,
399
431
  }
400
432
  throwIfAborted(command.signal)
401
- return finalizeProjectionMaterialization(storage, session, command, ready, result)
433
+ return {
434
+ kind: "committed",
435
+ result: await finalizeProjectionMaterialization(
436
+ storage,
437
+ session,
438
+ command,
439
+ ready,
440
+ commit,
441
+ result
442
+ ),
443
+ }
444
+ }
445
+
446
+ /**
447
+ * Refreshes the plan within the commit transaction. When it finds no more stale identities than
448
+ * the transaction may plan itself, they are planned here: nothing can change them again before the
449
+ * commit, so the second refresh finds the plan fresh.
450
+ */
451
+ async function refreshProjectionPlan(
452
+ context: MaterializerContext,
453
+ storage: MaterializerStorage,
454
+ command: PreparedProjectionReplacement,
455
+ commit: OntologyCommitWrite,
456
+ plan: ReplacementPlanRef
457
+ ): Promise<ReplacementPlanStatus> {
458
+ const plans = storage.ontology.replacementPlans
459
+ const status = await plans.refresh(plan)
460
+ if (status.fresh || status.unplanned > context.batching.transactionReplanRows) return status
461
+ await planProjectionReplacement(context, plans, plan, planInput(command, commit))
462
+ return plans.refresh(plan)
402
463
  }
403
464
 
404
465
  async function finalizeProjectionMaterialization(
@@ -406,12 +467,13 @@ async function finalizeProjectionMaterialization(
406
467
  session: Parameters<OntologyMaterializationStorage["finalize"]>[0]["session"],
407
468
  command: PreparedProjectionReplacement,
408
469
  ready: ReadyProjectionReplacement,
470
+ commit: OntologyCommitWrite,
409
471
  result: ProjectionCommitResult
410
472
  ): Promise<ProjectionCommitResult> {
411
473
  const applied = await storage.ontology.materializations.finalize({
412
474
  session,
413
475
  finalization: {
414
- sourceActivations: [projectionActivation(command, ready)],
476
+ sourceActivations: [projectionActivation(command, ready, commit)],
415
477
  result,
416
478
  },
417
479
  })
@@ -431,8 +493,7 @@ function projectionOrigin(command: PreparedProjectionReplacement): OntologyMater
431
493
  function projectionCommit(
432
494
  projectId: string,
433
495
  command: PreparedProjectionReplacement,
434
- identity: TimedCommitIdentity,
435
- origin: OntologyMaterializationOrigin
496
+ identity: TimedCommitIdentity
436
497
  ): OntologyCommitWrite {
437
498
  return {
438
499
  projectId,
@@ -440,8 +501,7 @@ function projectionCommit(
440
501
  idempotencyKey: identity.idempotencyKey,
441
502
  requestHash: identity.requestHash,
442
503
  executionId: command.scopeExecution.executionId,
443
- origin,
444
- ...command.scopeExecution.attribution,
504
+ origin: projectionOrigin(command),
445
505
  ontologyRevision: command.runIdentity.ontologyRevision,
446
506
  projectionRevision: command.runIdentity.projectionRevision,
447
507
  ownershipHash: command.runIdentity.ownershipHash,
@@ -450,35 +510,18 @@ function projectionCommit(
450
510
  }
451
511
  }
452
512
 
453
- async function planReadyProjection(
454
- context: MaterializerContext,
455
- storage: OntologyMaterializationStorage,
456
- session: Parameters<OntologyMaterializationStorage["finalize"]>[0]["session"],
513
+ function planInput(
457
514
  command: PreparedProjectionReplacement,
458
- ready: ReadyProjectionReplacement,
459
- origin: OntologyMaterializationOrigin
460
- ) {
461
- const input = {
462
- source: command.source,
463
- materializationId: ready.materializationId,
464
- projectionKind: command.projectionKind,
465
- identity: ready.identity,
466
- origin,
467
- correlationId: command.scopeExecution.correlationId,
468
- attribution: command.scopeExecution.attribution,
469
- }
470
- if (command.signal === undefined) {
471
- return planProjectionReplacement(context, storage, session, input)
472
- }
473
- return planProjectionReplacement(context, storage, session, {
474
- ...input,
475
- signal: command.signal,
476
- })
515
+ commit: OntologyCommitWrite
516
+ ): ProjectionReplacementPlanInput {
517
+ const input = { projectionKind: command.projectionKind, commit }
518
+ return command.signal === undefined ? input : { ...input, signal: command.signal }
477
519
  }
478
520
 
479
521
  function projectionActivation(
480
522
  command: PreparedProjectionReplacement,
481
- ready: ReadyProjectionReplacement
523
+ ready: ReadyProjectionReplacement,
524
+ commit: OntologyCommitWrite
482
525
  ) {
483
526
  return {
484
527
  source: command.source,
@@ -491,8 +534,8 @@ function projectionActivation(
491
534
  ownershipHash: command.runIdentity.ownershipHash,
492
535
  ontologyRevision: command.runIdentity.ontologyRevision,
493
536
  expected: ready.expectedSource,
494
- lastCommitId: ready.identity.commitId,
495
- updatedAt: ready.identity.committedAt,
537
+ lastCommitId: commit.id,
538
+ updatedAt: commit.committedAt,
496
539
  }
497
540
  }
498
541
 
@@ -1,26 +1,22 @@
1
- import type {
2
- EffectiveChangeCounts,
3
- EffectiveLinkChange,
4
- EffectiveObjectChange,
5
- OntologyMaterializationOrigin,
6
- OntologyObjectRef,
7
- ProjectionSourceRef,
8
- } from "../../materialization/model"
1
+ import type { EffectiveLinkChange, EffectiveObjectChange } from "../../materialization/model"
9
2
  import {
10
3
  linkRefKey,
11
4
  linkRefSortKey,
12
5
  linkScopeSortKey,
13
6
  objectRefKey,
14
7
  objectRefSortKey,
8
+ utf8JsonByteLength,
15
9
  } from "../../materialization/refs"
16
10
  import type {
17
11
  MaterializationPlanWorkItem,
18
- MaterializationSession,
19
12
  MaterializationWorkRecord,
20
- OntologyMaterializationStorage,
13
+ OntologyCommitWrite,
14
+ OntologyReplacementPlanStorage,
15
+ PlannedReplacementIdentity,
16
+ ReplacementPlanRef,
21
17
  SourceReplacementLinkState,
22
18
  SourceReplacementObjectState,
23
- SourceReplacementStatePage,
19
+ StageReplacementPlanInput,
24
20
  } from "../../storage/ontology"
25
21
  import type { MaterializerContext } from "../context"
26
22
  import {
@@ -28,7 +24,6 @@ import {
28
24
  buildObjectMaterializationEventDraft,
29
25
  } from "../effective/build-events"
30
26
  import { diffEffectiveLink, diffEffectiveObject } from "../effective/diff"
31
- import { oneStateRequest } from "../effective/load-state"
32
27
  import {
33
28
  resolveEffectiveLink,
34
29
  resolveEffectiveLinkSlotMember,
@@ -37,8 +32,6 @@ import {
37
32
  usableLinkSlotOverride,
38
33
  } from "../effective/resolve"
39
34
  import { validateEffectiveObject } from "../effective/validate"
40
- import type { MaterializerAttribution } from "../execution/scope"
41
- import { stageWorkBounded, validateStagedCardinality } from "../execution/work-executor"
42
35
  import {
43
36
  appendEffectiveLinkWork,
44
37
  appendEffectiveObjectWork,
@@ -47,72 +40,84 @@ import {
47
40
  planWork,
48
41
  } from "../execution/work-records"
49
42
  import { throwIfAborted } from "../shared/abort"
50
- import type { TimedCommitIdentity } from "../shared/identity"
43
+ import { chunkBySize } from "../shared/chunking"
51
44
 
52
45
  export interface ProjectionReplacementPlanInput {
53
- readonly source: ProjectionSourceRef
54
- readonly materializationId: string
55
46
  readonly projectionKind: "object" | "link"
56
- readonly identity: TimedCommitIdentity
57
- readonly origin: OntologyMaterializationOrigin
58
- readonly correlationId: string
59
- readonly attribution: MaterializerAttribution
47
+ /** The commit the plan carries, timed as the plan was opened. */
48
+ readonly commit: OntologyCommitWrite
60
49
  readonly signal?: AbortSignal
61
50
  }
62
51
 
52
+ /**
53
+ * Plans every identity of the candidate's durable plan still to plan, outside any transaction.
54
+ * Objects come first: a link resolves against the existence its endpoints are planned to have.
55
+ */
63
56
  export async function planProjectionReplacement(
64
57
  context: MaterializerContext,
65
- storage: OntologyMaterializationStorage,
66
- session: MaterializationSession,
58
+ plans: OntologyReplacementPlanStorage,
59
+ plan: ReplacementPlanRef,
67
60
  input: ProjectionReplacementPlanInput
68
- ): Promise<EffectiveChangeCounts> {
69
- const counts = emptyCounts()
61
+ ): Promise<void> {
62
+ const staged = { ...plan, commit: input.commit }
70
63
  if (input.projectionKind === "object") {
71
- await planObjectReplacement(context, storage, session, input, counts)
64
+ for await (const page of plans.streamState({
65
+ ...plan,
66
+ entityKind: "object",
67
+ pageRows: context.batching.statePageRows,
68
+ })) {
69
+ throwIfAborted(input.signal)
70
+ await stagePlanned(
71
+ context,
72
+ plans,
73
+ staged,
74
+ page.objects.map((state) => ({
75
+ identity: { kind: "object", ref: state.ref },
76
+ records: planReplacementObject(context, input, state),
77
+ }))
78
+ )
79
+ }
72
80
  }
73
- await planLinkReplacement(context, storage, session, input, counts)
74
- await validateStagedCardinality(context, storage, session, input.signal)
75
- return counts
76
- }
77
-
78
- async function planObjectReplacement(
79
- context: MaterializerContext,
80
- storage: OntologyMaterializationStorage,
81
- session: MaterializationSession,
82
- input: ProjectionReplacementPlanInput,
83
- counts: MutableCounts
84
- ): Promise<void> {
85
- for await (const page of storage.streamSourceReplacementState({
86
- session,
87
- source: input.source,
88
- candidateMaterializationId: input.materializationId,
89
- entityKind: "object",
81
+ for await (const page of plans.streamState({
82
+ ...plan,
83
+ entityKind: "link",
90
84
  pageRows: context.batching.statePageRows,
91
85
  })) {
92
86
  throwIfAborted(input.signal)
93
- const work = planObjectPage(context, input, page, counts)
94
- await stageWorkBounded(context, storage, session, work)
87
+ const endpoints = new Map(
88
+ page.endpoints.map((endpoint) => [objectRefKey(endpoint.ref), endpoint.exists] as const)
89
+ )
90
+ await stagePlanned(
91
+ context,
92
+ plans,
93
+ staged,
94
+ page.links.map((state) => ({
95
+ identity: { kind: "link", ref: state.ref },
96
+ records: planReplacementLink(context, input, state, endpoints),
97
+ }))
98
+ )
95
99
  }
96
100
  }
97
101
 
98
- function planObjectPage(
99
- context: MaterializerContext,
100
- input: ProjectionReplacementPlanInput,
101
- page: SourceReplacementStatePage,
102
- counts: MutableCounts
103
- ): MaterializationWorkRecord[] {
104
- const work: MaterializationWorkRecord[] = []
105
- for (const state of page.objects) {
106
- work.push(...planReplacementObject(context, input, state, counts))
102
+ async function stagePlanned(
103
+ context: Pick<MaterializerContext, "batching">,
104
+ plans: OntologyReplacementPlanStorage,
105
+ plan: Omit<StageReplacementPlanInput, "planned">,
106
+ planned: readonly PlannedReplacementIdentity[]
107
+ ): Promise<void> {
108
+ for await (const chunk of chunkBySize(planned, {
109
+ maxRows: context.batching.planChunkRows,
110
+ maxBytes: context.batching.planChunkBytes,
111
+ byteLength: (value) => utf8JsonByteLength(value.records),
112
+ })) {
113
+ await plans.stage({ ...plan, planned: chunk })
107
114
  }
108
- return work
109
115
  }
110
116
 
111
117
  function planReplacementObject(
112
118
  context: MaterializerContext,
113
119
  input: ProjectionReplacementPlanInput,
114
- state: SourceReplacementObjectState,
115
- counts: MutableCounts
120
+ state: SourceReplacementObjectState
116
121
  ): MaterializationWorkRecord[] {
117
122
  const resolved = resolveReplacementObject(context, state)
118
123
  if (resolved) validateEffectiveObject(context.ontology, resolved.ref, resolved.properties)
@@ -120,8 +125,8 @@ function planReplacementObject(
120
125
  const change = diffEffectiveObject({
121
126
  before: state.effective,
122
127
  resolved,
123
- commitId: input.identity.commitId,
124
- committedAt: input.identity.committedAt,
128
+ commitId: input.commit.id,
129
+ committedAt: input.commit.committedAt,
125
130
  })
126
131
  const sortKey = objectRefSortKey(state.ref)
127
132
  const work: MaterializationWorkRecord[] = [
@@ -137,131 +142,27 @@ function planReplacementObject(
137
142
  work.push({ kind: "incident-object", recordKey: `incident:${sortKey}`, ref: state.ref })
138
143
  }
139
144
 
140
- if (!change) {
141
- counts.objectsUnchanged += 1
142
- return work
143
- }
144
-
145
- incrementObjectCount(counts, change.kind)
146
- appendObjectChangeWork(work, sortKey, context, input, change)
145
+ if (!change) return work
146
+ appendObjectChangeWork(work, sortKey, change)
147
147
  return work
148
148
  }
149
149
 
150
150
  function appendObjectChangeWork(
151
151
  work: MaterializationWorkRecord[],
152
152
  sortKey: string,
153
- context: Pick<MaterializerContext, "projectId">,
154
- input: ProjectionReplacementPlanInput,
155
153
  change: EffectiveObjectChange
156
154
  ): void {
157
155
  const items: MaterializationPlanWorkItem[] = []
158
156
  appendEffectiveObjectWork(items, change)
159
157
  for (const item of items) work.push(planWork(item, sortKey))
160
- work.push(
161
- eventWork(
162
- buildObjectMaterializationEventDraft({
163
- projectId: context.projectId,
164
- commitId: input.identity.commitId,
165
- committedAt: input.identity.committedAt,
166
- origin: input.origin,
167
- correlationId: input.correlationId,
168
- attribution: input.attribution,
169
- change,
170
- })
171
- )
172
- )
173
- }
174
-
175
- async function planLinkReplacement(
176
- context: MaterializerContext,
177
- storage: OntologyMaterializationStorage,
178
- session: MaterializationSession,
179
- input: ProjectionReplacementPlanInput,
180
- counts: MutableCounts
181
- ): Promise<void> {
182
- for await (const page of storage.streamSourceReplacementState({
183
- session,
184
- source: input.source,
185
- candidateMaterializationId: input.materializationId,
186
- entityKind: "link",
187
- pageRows: context.batching.statePageRows,
188
- })) {
189
- throwIfAborted(input.signal)
190
- const endpointExistence = await loadEndpointExistence(context, storage, session, page)
191
- const work = planLinkPage(context, input, page, endpointExistence, counts)
192
- await stageWorkBounded(context, storage, session, work)
193
- }
194
- }
195
-
196
- async function loadEndpointExistence(
197
- context: MaterializerContext,
198
- storage: OntologyMaterializationStorage,
199
- session: MaterializationSession,
200
- page: SourceReplacementStatePage
201
- ): Promise<ReadonlyMap<string, boolean>> {
202
- const endpointRefs = collectEndpointRefs(page.links)
203
- const endpointExistence = new Map<string, boolean>()
204
- for (const value of await storage.readObjectExistence({
205
- session,
206
- refs: [...endpointRefs.values()],
207
- })) {
208
- endpointExistence.set(objectRefKey(value.ref), value.exists)
209
- }
210
-
211
- const missingEndpoints = [...endpointRefs.values()].filter(
212
- (ref) => !endpointExistence.has(objectRefKey(ref))
213
- )
214
- if (missingEndpoints.length === 0) return endpointExistence
215
-
216
- for await (const endpointPage of storage.streamState({
217
- session,
218
- requests: oneStateRequest({
219
- objects: missingEndpoints,
220
- links: [],
221
- linkScopes: [],
222
- incidentObjects: [],
223
- points: [],
224
- }),
225
- pageRows: context.batching.statePageRows,
226
- })) {
227
- for (const endpoint of endpointPage.objects) {
228
- endpointExistence.set(objectRefKey(endpoint.ref), endpoint.effective !== null)
229
- }
230
- }
231
- return endpointExistence
232
- }
233
-
234
- function collectEndpointRefs(
235
- links: readonly SourceReplacementLinkState[]
236
- ): ReadonlyMap<string, OntologyObjectRef> {
237
- const refs = new Map<string, OntologyObjectRef>()
238
- for (const state of links) {
239
- refs.set(objectRefKey(state.ref.source), state.ref.source)
240
- refs.set(objectRefKey(state.ref.target), state.ref.target)
241
- }
242
- return refs
243
- }
244
-
245
- function planLinkPage(
246
- context: MaterializerContext,
247
- input: ProjectionReplacementPlanInput,
248
- page: SourceReplacementStatePage,
249
- endpointExistence: ReadonlyMap<string, boolean>,
250
- counts: MutableCounts
251
- ): MaterializationWorkRecord[] {
252
- const work: MaterializationWorkRecord[] = []
253
- for (const state of page.links) {
254
- work.push(...planReplacementLink(context, input, state, endpointExistence, counts))
255
- }
256
- return work
158
+ work.push(eventWork(buildObjectMaterializationEventDraft(change)))
257
159
  }
258
160
 
259
161
  function planReplacementLink(
260
162
  context: MaterializerContext,
261
163
  input: ProjectionReplacementPlanInput,
262
164
  state: SourceReplacementLinkState,
263
- endpointExistence: ReadonlyMap<string, boolean>,
264
- counts: MutableCounts
165
+ endpointExistence: ReadonlyMap<string, boolean>
265
166
  ): MaterializationWorkRecord[] {
266
167
  const resolved = resolveReplacementLink(context, state, endpointExistence)
267
168
  const work = cardinalityWork(context, state, resolved !== null)
@@ -270,18 +171,13 @@ function planReplacementLink(
270
171
  const change = diffEffectiveLink({
271
172
  before: state.effective,
272
173
  resolved,
273
- commitId: input.identity.commitId,
274
- committedAt: input.identity.committedAt,
174
+ commitId: input.commit.id,
175
+ committedAt: input.commit.committedAt,
275
176
  })
276
177
  const sortKey = linkRefSortKey(state.ref)
277
178
  work.push(classificationWork("link", linkRefKey(state.ref), sortKey))
278
- if (!change) {
279
- counts.linksUnchanged += 1
280
- return work
281
- }
282
-
283
- incrementLinkCount(counts, change.kind)
284
- appendLinkChangeWork(work, sortKey, context, input, change)
179
+ if (!change) return work
180
+ appendLinkChangeWork(work, sortKey, change)
285
181
  return work
286
182
  }
287
183
 
@@ -348,26 +244,12 @@ function cardinalityWork(
348
244
  function appendLinkChangeWork(
349
245
  work: MaterializationWorkRecord[],
350
246
  sortKey: string,
351
- context: Pick<MaterializerContext, "projectId">,
352
- input: ProjectionReplacementPlanInput,
353
247
  change: EffectiveLinkChange
354
248
  ): void {
355
249
  const items: MaterializationPlanWorkItem[] = []
356
250
  appendEffectiveLinkWork(items, change)
357
251
  for (const item of items) work.push(planWork(item, sortKey))
358
- work.push(
359
- eventWork(
360
- buildLinkMaterializationEventDraft({
361
- projectId: context.projectId,
362
- commitId: input.identity.commitId,
363
- committedAt: input.identity.committedAt,
364
- origin: input.origin,
365
- correlationId: input.correlationId,
366
- attribution: input.attribution,
367
- change,
368
- })
369
- )
370
- )
252
+ work.push(eventWork(buildLinkMaterializationEventDraft(change)))
371
253
  }
372
254
 
373
255
  function resolveReplacementObject(
@@ -383,30 +265,3 @@ function resolveReplacementObject(
383
265
  latestTelemetry: state.latestTelemetry,
384
266
  })
385
267
  }
386
-
387
- function emptyCounts(): MutableCounts {
388
- return {
389
- objectsCreated: 0,
390
- objectsUpdated: 0,
391
- objectsDeleted: 0,
392
- objectsUnchanged: 0,
393
- linksCreated: 0,
394
- linksUpdated: 0,
395
- linksDeleted: 0,
396
- linksUnchanged: 0,
397
- }
398
- }
399
-
400
- type MutableCounts = { -readonly [K in keyof EffectiveChangeCounts]: EffectiveChangeCounts[K] }
401
-
402
- function incrementObjectCount(counts: MutableCounts, kind: EffectiveObjectChange["kind"]): void {
403
- if (kind === "created") counts.objectsCreated += 1
404
- else if (kind === "updated") counts.objectsUpdated += 1
405
- else counts.objectsDeleted += 1
406
- }
407
-
408
- function incrementLinkCount(counts: MutableCounts, kind: EffectiveLinkChange["kind"]): void {
409
- if (kind === "created") counts.linksCreated += 1
410
- else if (kind === "updated") counts.linksUpdated += 1
411
- else counts.linksDeleted += 1
412
- }