@chkit/plugin-ingest 0.2.0-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +43 -0
  2. package/dist/destination.d.ts +40 -0
  3. package/dist/destination.d.ts.map +1 -0
  4. package/dist/destination.js +65 -0
  5. package/dist/destination.js.map +1 -0
  6. package/dist/errors.d.ts +33 -0
  7. package/dist/errors.d.ts.map +1 -0
  8. package/dist/errors.js +133 -0
  9. package/dist/errors.js.map +1 -0
  10. package/dist/executor.d.ts +30 -0
  11. package/dist/executor.d.ts.map +1 -0
  12. package/dist/executor.js +552 -0
  13. package/dist/executor.js.map +1 -0
  14. package/dist/incremental.d.ts +45 -0
  15. package/dist/incremental.d.ts.map +1 -0
  16. package/dist/incremental.js +72 -0
  17. package/dist/incremental.js.map +1 -0
  18. package/dist/index.d.ts +12 -0
  19. package/dist/index.d.ts.map +1 -0
  20. package/dist/index.js +11 -0
  21. package/dist/index.js.map +1 -0
  22. package/dist/journal-history.d.ts +15 -0
  23. package/dist/journal-history.d.ts.map +1 -0
  24. package/dist/journal-history.js +366 -0
  25. package/dist/journal-history.js.map +1 -0
  26. package/dist/journal-records.d.ts +28 -0
  27. package/dist/journal-records.d.ts.map +1 -0
  28. package/dist/journal-records.js +76 -0
  29. package/dist/journal-records.js.map +1 -0
  30. package/dist/journal.d.ts +16 -0
  31. package/dist/journal.d.ts.map +1 -0
  32. package/dist/journal.js +72 -0
  33. package/dist/journal.js.map +1 -0
  34. package/dist/loader.d.ts +12 -0
  35. package/dist/loader.d.ts.map +1 -0
  36. package/dist/loader.js +37 -0
  37. package/dist/loader.js.map +1 -0
  38. package/dist/paginate.d.ts +24 -0
  39. package/dist/paginate.d.ts.map +1 -0
  40. package/dist/paginate.js +37 -0
  41. package/dist/paginate.js.map +1 -0
  42. package/dist/plugin.d.ts +155 -0
  43. package/dist/plugin.d.ts.map +1 -0
  44. package/dist/plugin.js +353 -0
  45. package/dist/plugin.js.map +1 -0
  46. package/dist/queue.d.ts +9 -0
  47. package/dist/queue.d.ts.map +1 -0
  48. package/dist/queue.js +47 -0
  49. package/dist/queue.js.map +1 -0
  50. package/dist/recovery.d.ts +41 -0
  51. package/dist/recovery.d.ts.map +1 -0
  52. package/dist/recovery.js +145 -0
  53. package/dist/recovery.js.map +1 -0
  54. package/dist/registry.d.ts +48 -0
  55. package/dist/registry.d.ts.map +1 -0
  56. package/dist/registry.js +109 -0
  57. package/dist/registry.js.map +1 -0
  58. package/dist/retry.d.ts +12 -0
  59. package/dist/retry.d.ts.map +1 -0
  60. package/dist/retry.js +69 -0
  61. package/dist/retry.js.map +1 -0
  62. package/dist/testing.d.ts +18 -0
  63. package/dist/testing.d.ts.map +1 -0
  64. package/dist/testing.js +54 -0
  65. package/dist/testing.js.map +1 -0
  66. package/dist/types.d.ts +229 -0
  67. package/dist/types.d.ts.map +1 -0
  68. package/dist/types.js +2 -0
  69. package/dist/types.js.map +1 -0
  70. package/package.json +56 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"recovery.js","sourceRoot":"","sources":["../src/recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAExC,OAAO,EAAE,YAAY,EAAoD,MAAM,mBAAmB,CAAA;AAElG,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAA;AAC7D,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,EAAE,eAAe,EAAmB,MAAM,cAAc,CAAA;AAG7G,MAAM,UAAU,GAAG,0BAA0B,CAAA;AAC7C,MAAM,aAAa,GAAG;IACpB,6BAA6B,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,uCAAuC,EAAE,CAAC;CACtE,CAAA;AAoC9B,iFAAiF;AACjF,MAAM,UAAU,iBAAiB,CAC/B,QAA+B,EAC/B,WAAmB,EACnB,WAAmB,EACnB,QAAiB;IAEjB,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAA;IAC9D,MAAM,SAAS,GAAG,sBAAsB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IAC/D,MAAM,WAAW,GAAG,iBAAiB,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAA;IACnF,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,gBAAgB,GAAG,GAAG,WAAW,WAAW,MAAM,EAAE,CAAA;IAC1D,OAAO;QACL,WAAW;QACX,WAAW;QACX,WAAW;QACX,OAAO,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QACxC,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,YAAY,EAAE,QAAQ,CAAC,MAAM;QAC7B,oBAAoB,EAAE,QAAQ,CAAC,MAAM;QACrC,aAAa,EAAE,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM;QACxE,IAAI,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACrD,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,YAAY,EAAE,GAAG,WAAW,YAAY,MAAM,EAAE;QAChD,gBAAgB;QAChB,UAAU,EAAE,uGAAuG,gBAAgB,8BAA8B;QACjK,aAAa,EAAE,4HAA4H;KAC5I,CAAA;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,WAAmB,EAAE,OAA+B;IACtF,MAAM,KAAK,GAAG,aAAa,CAAC,OAAO,CAAC,CAAA;IACpC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IACpD,OAAO,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA;AACnE,CAAC;AAED,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,WAAmB,EACnB,WAA+B,EAC/B,OAA+B;IAE/B,MAAM,KAAK,GAAG,aAAa,CAAC,OAAO,CAAC,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAClD,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA;IACrE,IAAI,WAAW,KAAK,SAAS;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;IAChF,IAAI,WAAW,KAAK,IAAI,CAAC,WAAW;QAAE,MAAM,IAAI,iBAAiB,CAAC,0GAA0G,CAAC,CAAA;IAC7K,IAAI,CAAC,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1E,IAAI,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;IAEnE,+EAA+E;IAC/E,gFAAgF;IAChF,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAC5D,MAAM,gBAAgB,GAAG,GAAG,IAAI,CAAC,gBAAgB,IAAI,MAAM,EAAE,CAAA;IAC7D,MAAM,gBAAgB,GAAG;QACvB,GAAG,IAAI;QACP,YAAY,EAAE,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,EAAE;QAC9C,gBAAgB;QAChB,UAAU,EAAE,yEAAyE,gBAAgB,mEAAmE;KACzK,CAAA;IACD,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAC3E,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA;IACzE,MAAM,SAAS,GAAG,sBAAsB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IAC/D,MAAM,QAAQ,GAAG;QACf,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/E,GAAG,SAAS,CAAC,IAAI;KAClB,CAAA;IACD,MAAM,gBAAgB,CAAC,gBAAgB,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC3D,MAAM,MAAM,GAAG,sBAAsB,CAAC,YAAY,CAAC,MAAM,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,CAAA;IAC9I,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QACtG,MAAM,IAAI,KAAK,CAAC,gFAAgF,CAAC,CAAA;IACnG,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IACxD,IAAI,iBAAiB,CAAC,aAAa,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,WAAW,EAAE,CAAC;QAC3F,MAAM,IAAI,iBAAiB,CAAC,kHAAkH,CAAC,CAAA;IACjJ,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAA;AACpE,CAAC;AAED,SAAS,aAAa,CAAC,OAA+B;IACpD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,qBAAqB,CAAA;IACpD,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC5B,UAAU,CAAC,KAAK,CAAC,CAAA;IACjB,OAAO,KAAK,CAAA;AACd,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,KAAa,EAAE,OAA+B;IACzE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,KAAK,CACzC,qDAAqD,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,SAAS,CAAC,KAAK,CAAC,EAAE,EACjH,aAAa,CACd,CAAA;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;AACrE,CAAC;AAED,SAAS,UAAU,CAAC,QAAkD,EAAE,WAAmB,EAAE,KAAa,EAAE,QAAgB;IAC1H,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;IAC3E,IAAI,CAAC,QAAQ,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAClC,OAAO;QACL,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK;QAC1C,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,wBAAwB,KAAK,sFAAsF,CAAC;KAClJ,CAAA;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAa,EAAE,OAA+B;IACxE,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,CAC3B,iBAAiB,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,wFAAwF,EAC3I,aAAa,CACd,CAAA;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,IAA2B,EAAE,OAA+B;IACzG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CAAA;IACnF,MAAM,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5B,KAAK,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,KAAK,EAAE;YACrC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC;YACjB,QAAQ,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,0BAA0B,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE;SACrF,CAAC,CAAA;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IACnD,IAAI,mBAAmB,CAAC,QAAQ,CAAC,KAAK,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,oCAAoC,KAAK,yCAAyC,CAAC,CAAA;IACrG,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAA2B,EAAE,WAAmB,EAAE,QAA4B;IAClG,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAA;AACrE,CAAC;AAED,SAAS,UAAU,CAAC,GAAe,EAAE,WAAmB,EAAE,QAA4B;IACpF,OAAO,GAAG,CAAC,YAAY,KAAK,WAAW,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAA;AACnG,CAAC;AAED,SAAS,iBAAiB,CAAC,IAA2B,EAAE,WAAmB,EAAE,KAAa,EAAE,QAA4B;IACtH,OAAO,MAAM,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,QAAQ,IAAI,EAAE,EAAE,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChF,CAAC;AAED,SAAS,mBAAmB,CAAC,IAA2B;IACtD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;AAC7D,CAAC;AAED,SAAS,SAAS,CAAC,QAAgB,EAAE,KAAa;IAChD,UAAU,CAAC,QAAQ,CAAC,CAAA;IACpB,UAAU,CAAC,KAAK,CAAC,CAAA;IACjB,OAAO,KAAK,QAAQ,QAAQ,KAAK,IAAI,CAAA;AACvC,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,iBAAiB,CAAC,+BAA+B,KAAK,IAAI,CAAC,CAAA;AACpG,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAA;AACjE,CAAC"}
@@ -0,0 +1,48 @@
1
+ import type { TableDefinition } from '@chkit/core';
2
+ import type { AnyStreamDefinition, ErrorClassifier, IncrementalStrategy, LoaderFactory, PipelineDefinition, ReadContext, RetryOptions, Row, SourceChunk, StreamBudget, StreamDefinition } from './types.js';
3
+ export interface StreamInput<TRow extends Row, TState, TSelection> {
4
+ id: string;
5
+ destination: TableDefinition;
6
+ tags?: readonly string[];
7
+ incremental: IncrementalStrategy<TState, TSelection>;
8
+ read: (context: ReadContext<TSelection, TState>) => AsyncIterable<SourceChunk<TRow, TState>>;
9
+ loader?: LoaderFactory;
10
+ retry?: RetryOptions;
11
+ batchSize?: number;
12
+ budget?: StreamBudget;
13
+ classifyError?: ErrorClassifier;
14
+ }
15
+ export interface FullSyncStreamInput<TRow extends Row> extends Omit<StreamInput<TRow, undefined, undefined>, 'incremental'> {
16
+ incremental?: undefined;
17
+ }
18
+ export interface PipelineInput {
19
+ id: string;
20
+ tags?: readonly string[];
21
+ streams: readonly AnyStreamDefinition[];
22
+ maxStreams?: number;
23
+ maxFetches?: number;
24
+ maxLoads?: number;
25
+ retry?: RetryOptions;
26
+ }
27
+ export interface SelectedStream {
28
+ stream: AnyStreamDefinition;
29
+ pipeline: PipelineDefinition;
30
+ effectiveTags: readonly string[];
31
+ }
32
+ export declare function defineStream<TRow extends Row>(input: FullSyncStreamInput<TRow>): StreamDefinition<TRow, undefined, undefined>;
33
+ export declare function defineStream<TRow extends Row, TState, TSelection>(input: StreamInput<TRow, TState, TSelection>): StreamDefinition<TRow, TState, TSelection>;
34
+ /**
35
+ * Define a non-durable named group of streams. Pipeline identity never
36
+ * participates in checkpoint or batch identity, so moving a stream between
37
+ * pipelines does not reset its state.
38
+ */
39
+ export declare function definePipeline(input: PipelineInput): PipelineDefinition;
40
+ /** Only exported pipeline values participate; aliases of the same value count once. */
41
+ export declare function collectPipelines(modules: readonly Record<string, unknown>[]): PipelineDefinition[];
42
+ /**
43
+ * Repeated exact case-sensitive tags with AND semantics over the union of
44
+ * pipeline and stream tags plus derived `pipeline:<id>` and `stream:<id>`.
45
+ * No filter selects the complete graph; an explicit empty selection throws.
46
+ */
47
+ export declare function selectStreams(pipelines: readonly PipelineDefinition[], tags: readonly string[]): SelectedStream[];
48
+ //# sourceMappingURL=registry.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,OAAO,KAAK,EACV,mBAAmB,EACnB,eAAe,EACf,mBAAmB,EACnB,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,YAAY,EACZ,GAAG,EACH,WAAW,EACX,YAAY,EACZ,gBAAgB,EACjB,MAAM,YAAY,CAAA;AAInB,MAAM,WAAW,WAAW,CAAC,IAAI,SAAS,GAAG,EAAE,MAAM,EAAE,UAAU;IAC/D,EAAE,EAAE,MAAM,CAAA;IACV,WAAW,EAAE,eAAe,CAAA;IAC5B,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACxB,WAAW,EAAE,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IACpD,IAAI,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,aAAa,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;IAC5F,MAAM,CAAC,EAAE,aAAa,CAAA;IACtB,KAAK,CAAC,EAAE,YAAY,CAAA;IACpB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,aAAa,CAAC,EAAE,eAAe,CAAA;CAChC;AAED,MAAM,WAAW,mBAAmB,CAAC,IAAI,SAAS,GAAG,CACnD,SAAQ,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,aAAa,CAAC;IACpE,WAAW,CAAC,EAAE,SAAS,CAAA;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACxB,OAAO,EAAE,SAAS,mBAAmB,EAAE,CAAA;IACvC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,KAAK,CAAC,EAAE,YAAY,CAAA;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,mBAAmB,CAAA;IAC3B,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,aAAa,EAAE,SAAS,MAAM,EAAE,CAAA;CACjC;AAED,wBAAgB,YAAY,CAAC,IAAI,SAAS,GAAG,EAAE,KAAK,EAAE,mBAAmB,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;AAC9H,wBAAgB,YAAY,CAAC,IAAI,SAAS,GAAG,EAAE,MAAM,EAAE,UAAU,EAC/D,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,GAC3C,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAA;AAuB7C;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,kBAAkB,CAsBvE;AAqBD,uFAAuF;AACvF,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,kBAAkB,EAAE,CAMlG;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,SAAS,kBAAkB,EAAE,EAAE,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,cAAc,EAAE,CAmBjH"}
@@ -0,0 +1,109 @@
1
+ import { IngestConfigError } from './errors.js';
2
+ import { fullSync } from './incremental.js';
3
+ const STREAM_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/;
4
+ export function defineStream(input) {
5
+ assertValidId('stream', input.id);
6
+ if (input.batchSize !== undefined && (!Number.isInteger(input.batchSize) || input.batchSize <= 0)) {
7
+ throw new IngestConfigError(`Stream "${input.id}": batchSize must be a positive integer.`);
8
+ }
9
+ return {
10
+ kind: 'ingest_stream',
11
+ id: input.id,
12
+ destination: input.destination,
13
+ tags: dedupe(input.tags ?? []),
14
+ incremental: input.incremental ?? fullSync(),
15
+ read: input.read,
16
+ loader: input.loader,
17
+ retry: input.retry,
18
+ batchSize: input.batchSize,
19
+ budget: input.budget,
20
+ classifyError: input.classifyError,
21
+ };
22
+ }
23
+ /**
24
+ * Define a non-durable named group of streams. Pipeline identity never
25
+ * participates in checkpoint or batch identity, so moving a stream between
26
+ * pipelines does not reset its state.
27
+ */
28
+ export function definePipeline(input) {
29
+ assertValidId('pipeline', input.id);
30
+ const pipeline = {
31
+ kind: 'ingest_pipeline',
32
+ id: input.id,
33
+ tags: dedupe(input.tags ?? []),
34
+ streams: [...input.streams],
35
+ maxStreams: positiveCeiling(input.id, 'maxStreams', input.maxStreams, 4),
36
+ maxFetches: positiveCeiling(input.id, 'maxFetches', input.maxFetches, 4),
37
+ maxLoads: positiveCeiling(input.id, 'maxLoads', input.maxLoads, 2),
38
+ retry: input.retry,
39
+ };
40
+ const seen = new Set();
41
+ for (const stream of pipeline.streams) {
42
+ if (seen.has(stream.id)) {
43
+ throw new IngestConfigError(`Pipeline "${pipeline.id}" lists stream "${stream.id}" more than once.`);
44
+ }
45
+ seen.add(stream.id);
46
+ }
47
+ return pipeline;
48
+ }
49
+ /** Validate identities within this graph, without process-wide registration. */
50
+ function validatePipelines(pipelines) {
51
+ const ids = new Set();
52
+ const owners = new Map();
53
+ for (const pipeline of pipelines) {
54
+ if (ids.has(pipeline.id))
55
+ throw new IngestConfigError(`Duplicate pipeline id "${pipeline.id}".`);
56
+ ids.add(pipeline.id);
57
+ for (const stream of pipeline.streams) {
58
+ const owner = owners.get(stream.id);
59
+ if (owner !== undefined) {
60
+ throw new IngestConfigError(`Stream id "${stream.id}" occurs in pipelines "${owner}" and "${pipeline.id}". Stream ids must be globally unique.`);
61
+ }
62
+ owners.set(stream.id, pipeline.id);
63
+ }
64
+ }
65
+ }
66
+ /** Only exported pipeline values participate; aliases of the same value count once. */
67
+ export function collectPipelines(modules) {
68
+ const pipelines = [...new Set(modules.flatMap((mod) => Object.values(mod).filter((value) => typeof value === 'object' && value !== null && 'kind' in value && value.kind === 'ingest_pipeline')))];
69
+ validatePipelines(pipelines);
70
+ return pipelines;
71
+ }
72
+ /**
73
+ * Repeated exact case-sensitive tags with AND semantics over the union of
74
+ * pipeline and stream tags plus derived `pipeline:<id>` and `stream:<id>`.
75
+ * No filter selects the complete graph; an explicit empty selection throws.
76
+ */
77
+ export function selectStreams(pipelines, tags) {
78
+ validatePipelines(pipelines);
79
+ const wanted = dedupe(tags);
80
+ const all = pipelines.flatMap((pipeline) => pipeline.streams.map((stream) => ({
81
+ stream,
82
+ pipeline,
83
+ effectiveTags: dedupe([...pipeline.tags, ...stream.tags, `pipeline:${pipeline.id}`, `stream:${stream.id}`]),
84
+ })));
85
+ if (wanted.length === 0)
86
+ return all;
87
+ const selected = all.filter((entry) => wanted.every((tag) => entry.effectiveTags.includes(tag)));
88
+ if (selected.length === 0) {
89
+ throw new IngestConfigError(`No stream matches every requested tag: ${wanted.map((tag) => `--tag ${tag}`).join(' ')}. Nothing was executed.`);
90
+ }
91
+ return selected;
92
+ }
93
+ function assertValidId(kind, id) {
94
+ if (!STREAM_ID_PATTERN.test(id)) {
95
+ throw new IngestConfigError(`Invalid ${kind} id "${id}". Use letters, digits, "_", ".", ":" or "-", starting with a letter or digit.`);
96
+ }
97
+ }
98
+ function positiveCeiling(pipelineId, name, value, fallback) {
99
+ if (value === undefined)
100
+ return fallback;
101
+ if (!Number.isInteger(value) || value <= 0) {
102
+ throw new IngestConfigError(`Pipeline "${pipelineId}": ${name} must be a positive integer.`);
103
+ }
104
+ return value;
105
+ }
106
+ function dedupe(values) {
107
+ return [...new Set(values)];
108
+ }
109
+ //# sourceMappingURL=registry.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"registry.js","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAe3C,MAAM,iBAAiB,GAAG,+BAA+B,CAAA;AAwCzD,MAAM,UAAU,YAAY,CAC1B,KAAoE;IAEpE,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IACjC,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,EAAE,CAAC;QAClG,MAAM,IAAI,iBAAiB,CAAC,WAAW,KAAK,CAAC,EAAE,0CAA0C,CAAC,CAAA;IAC5F,CAAC;IACD,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,QAAQ,EAAE;QAC5C,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,aAAa,EAAE,KAAK,CAAC,aAAa;KACnC,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,KAAoB;IACjD,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IACnC,MAAM,QAAQ,GAAuB;QACnC,IAAI,EAAE,iBAAiB;QACvB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC;QAC3B,UAAU,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,UAAU,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,QAAQ,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAA;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,iBAAiB,CAAC,aAAa,QAAQ,CAAC,EAAE,mBAAmB,MAAM,CAAC,EAAE,mBAAmB,CAAC,CAAA;QACtG,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACrB,CAAC;IAED,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,gFAAgF;AAChF,SAAS,iBAAiB,CAAC,SAAwC;IACjE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,iBAAiB,CAAC,0BAA0B,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAA;QAChG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;QACpB,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,MAAM,IAAI,iBAAiB,CACzB,cAAc,MAAM,CAAC,EAAE,0BAA0B,KAAK,UAAU,QAAQ,CAAC,EAAE,wCAAwC,CACpH,CAAA;YACH,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAA;QACpC,CAAC;IACH,CAAC;AACH,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,gBAAgB,CAAC,OAA2C;IAC1E,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAC9E,CAAC,KAAK,EAA+B,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB,CAC3I,CAAC,CAAC,CAAC,CAAA;IACJ,iBAAiB,CAAC,SAAS,CAAC,CAAA;IAC5B,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAwC,EAAE,IAAuB;IAC7F,iBAAiB,CAAC,SAAS,CAAC,CAAA;IAC5B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;IAC3B,MAAM,GAAG,GAAqB,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC3D,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM;QACN,QAAQ;QACR,aAAa,EAAE,MAAM,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE,YAAY,QAAQ,CAAC,EAAE,EAAE,EAAE,UAAU,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;KAC5G,CAAC,CAAC,CACJ,CAAA;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAA;IAEnC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAChG,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,iBAAiB,CACzB,0CAA0C,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,yBAAyB,CACjH,CAAA;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,IAA2B,EAAE,EAAU;IAC5D,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,iBAAiB,CACzB,WAAW,IAAI,QAAQ,EAAE,gFAAgF,CAC1G,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,UAAkB,EAAE,IAAY,EAAE,KAAyB,EAAE,QAAgB;IACpG,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAA;IACxC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,iBAAiB,CAAC,aAAa,UAAU,MAAM,IAAI,8BAA8B,CAAC,CAAA;IAC9F,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,MAAM,CAAC,MAAyB;IACvC,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;AAC7B,CAAC"}
@@ -0,0 +1,12 @@
1
+ import type { ErrorClassifier, RetryContext, RetryOptions } from './types.js';
2
+ interface AttemptEnv {
3
+ signal: AbortSignal;
4
+ classifier: ErrorClassifier | undefined;
5
+ onRetry: (context: RetryContext, retryAfterMs: number | undefined) => Promise<void> | void;
6
+ }
7
+ /** Pipeline default, partially overridden by the stream. */
8
+ export declare function mergeRetry(...layers: Array<RetryOptions | undefined>): RetryOptions;
9
+ /** p-retry owns retry budgets and backoff; this adapter adds provider classification and Retry-After. */
10
+ export declare function runAttempt<T>(operation: (attemptNumber: number) => Promise<T>, policy: RetryOptions, env: AttemptEnv): Promise<T>;
11
+ export {};
12
+ //# sourceMappingURL=retry.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAW7E,UAAU,UAAU;IAClB,MAAM,EAAE,WAAW,CAAA;IACnB,UAAU,EAAE,eAAe,GAAG,SAAS,CAAA;IACvC,OAAO,EAAE,CAAC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CAC3F;AAED,4DAA4D;AAC5D,wBAAgB,UAAU,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,GAAG,YAAY,CASnF;AAED,yGAAyG;AACzG,wBAAgB,UAAU,CAAC,CAAC,EAC1B,SAAS,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EAChD,MAAM,EAAE,YAAY,EACpB,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,CAAC,CAAC,CAqCZ"}
package/dist/retry.js ADDED
@@ -0,0 +1,69 @@
1
+ import { setTimeout as sleep } from 'node:timers/promises';
2
+ import pRetry, { AbortError } from 'p-retry';
3
+ import { classifyFailure, FetchFailure, IngestConfigError } from './errors.js';
4
+ const DEFAULT_RETRY = {
5
+ retries: 5,
6
+ factor: 2,
7
+ minTimeout: 1000,
8
+ maxTimeout: 60_000,
9
+ randomize: true,
10
+ maxRetryTime: 10 * 60_000,
11
+ };
12
+ /** Pipeline default, partially overridden by the stream. */
13
+ export function mergeRetry(...layers) {
14
+ const merged = {};
15
+ for (const layer of layers) {
16
+ if (!layer)
17
+ continue;
18
+ for (const [key, value] of Object.entries(layer)) {
19
+ if (value !== undefined)
20
+ Object.assign(merged, { [key]: value });
21
+ }
22
+ }
23
+ return merged;
24
+ }
25
+ /** p-retry owns retry budgets and backoff; this adapter adds provider classification and Retry-After. */
26
+ export function runAttempt(operation, policy, env) {
27
+ const options = { ...DEFAULT_RETRY, ...mergeRetry(policy) };
28
+ const deadline = performance.now() + options.maxRetryTime;
29
+ let notBefore = 0;
30
+ return pRetry(async (attemptNumber) => {
31
+ // Backoff and Retry-After overlap: only wait out the remaining provider embargo.
32
+ const remaining = notBefore - performance.now();
33
+ if (remaining > 0)
34
+ await sleep(remaining, undefined, { signal: env.signal });
35
+ env.signal.throwIfAborted();
36
+ try {
37
+ return await operation(attemptNumber);
38
+ }
39
+ catch (cause) {
40
+ // A nested source attempt already exhausted its retries. Never replay it
41
+ // through the reader's coarser retry boundary.
42
+ if (cause instanceof FetchFailure || cause instanceof IngestConfigError)
43
+ throw new AbortError(cause);
44
+ const failure = new FetchFailure(cause, classifyFailure(cause, env.signal, env.classifier));
45
+ if (failure.classification.kind === 'cancelled' || failure.classification.kind === 'permanent')
46
+ throw new AbortError(failure);
47
+ throw failure;
48
+ }
49
+ }, {
50
+ ...options,
51
+ signal: env.signal,
52
+ shouldConsumeRetry: ({ error, ...context }) => error instanceof FetchFailure ? (policy.shouldConsumeRetry?.({ ...context, error }) ?? true) : true,
53
+ shouldRetry: async ({ error, ...context }) => {
54
+ env.signal.throwIfAborted();
55
+ if (!(error instanceof FetchFailure))
56
+ return false;
57
+ const retryContext = { ...context, error };
58
+ if (policy.shouldRetry && !(await policy.shouldRetry(retryContext)))
59
+ return false;
60
+ const retryAfterMs = error.classification.kind === 'rate_limited' ? error.classification.retryAfterMs : undefined;
61
+ notBefore = performance.now() + (retryAfterMs ?? 0);
62
+ if (notBefore > deadline)
63
+ return false;
64
+ await env.onRetry(retryContext, retryAfterMs);
65
+ return true;
66
+ },
67
+ });
68
+ }
69
+ //# sourceMappingURL=retry.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retry.js","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAE5C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAG9E,MAAM,aAAa,GAAG;IACpB,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,CAAC;IACT,UAAU,EAAE,IAAI;IAChB,UAAU,EAAE,MAAM;IAClB,SAAS,EAAE,IAAI;IACf,YAAY,EAAE,EAAE,GAAG,MAAM;CAC1B,CAAA;AAQD,4DAA4D;AAC5D,MAAM,UAAU,UAAU,CAAC,GAAG,MAAuC;IACnE,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACjD,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;QAClE,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,UAAU,CACxB,SAAgD,EAChD,MAAoB,EACpB,GAAe;IAEf,MAAM,OAAO,GAAG,EAAE,GAAG,aAAa,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,CAAA;IAC3D,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,YAAY,CAAA;IACzD,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,OAAO,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE;QACpC,iFAAiF;QACjF,MAAM,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QAC/C,IAAI,SAAS,GAAG,CAAC;YAAE,MAAM,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;QAC5E,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAC3B,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,CAAC,aAAa,CAAC,CAAA;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yEAAyE;YACzE,+CAA+C;YAC/C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,iBAAiB;gBAAE,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,CAAA;YACpG,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAA;YAC3F,IAAI,OAAO,CAAC,cAAc,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,cAAc,CAAC,IAAI,KAAK,WAAW;gBAAE,MAAM,IAAI,UAAU,CAAC,OAAO,CAAC,CAAA;YAC7H,MAAM,OAAO,CAAA;QACf,CAAC;IACH,CAAC,EAAE;QACD,GAAG,OAAO;QACV,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,kBAAkB,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,CAC5C,KAAK,YAAY,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;QACrG,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE;YAC3C,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;YAC3B,IAAI,CAAC,CAAC,KAAK,YAAY,YAAY,CAAC;gBAAE,OAAO,KAAK,CAAA;YAClD,MAAM,YAAY,GAAG,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,CAAA;YAC1C,IAAI,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAA;YACjF,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAA;YACjH,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,CAAA;YACnD,IAAI,SAAS,GAAG,QAAQ;gBAAE,OAAO,KAAK,CAAA;YACtC,MAAM,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,YAAY,CAAC,CAAA;YAC7C,OAAO,IAAI,CAAA;QACb,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
@@ -0,0 +1,18 @@
1
+ import { type JournalRow } from './journal.js';
2
+ import type { DestinationAdapter, Journal, JournalEvent, Row } from './types.js';
3
+ export interface MemoryJournal extends Journal {
4
+ readonly events: JournalEvent[];
5
+ readonly rows: JournalRow[];
6
+ }
7
+ export interface MemoryDestination extends DestinationAdapter {
8
+ /** Physical rows per `database.table`, after token deduplication. */
9
+ readonly tables: Map<string, Row[]>;
10
+ readonly tokens: string[];
11
+ }
12
+ /** In-memory journal with the same projection semantics as the ClickHouse one. */
13
+ export declare function createMemoryJournal(options?: {
14
+ now?: () => Date;
15
+ }): MemoryJournal;
16
+ /** In-memory destination that honours `insert_deduplication_token` like ClickHouse. */
17
+ export declare function createMemoryDestination(): MemoryDestination;
18
+ //# sourceMappingURL=testing.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AACA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,cAAc,CAAA;AAC5D,OAAO,KAAK,EAAuB,kBAAkB,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAErG,MAAM,WAAW,aAAc,SAAQ,OAAO;IAC5C,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,CAAA;IAC/B,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,iBAAkB,SAAQ,kBAAkB;IAC3D,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;IACnC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAC1B;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,OAAO,GAAE;IAAE,GAAG,CAAC,EAAE,MAAM,IAAI,CAAA;CAAO,GAAG,aAAa,CA+BrF;AAED,uFAAuF;AACvF,wBAAgB,uBAAuB,IAAI,iBAAiB,CAa3D"}
@@ -0,0 +1,54 @@
1
+ import { validateJournalHistory } from './journal-history.js';
2
+ import { toJournalRow } from './journal.js';
3
+ /** In-memory journal with the same projection semantics as the ClickHouse one. */
4
+ export function createMemoryJournal(options = {}) {
5
+ const events = [];
6
+ const rows = [];
7
+ const now = options.now ?? (() => new Date());
8
+ return {
9
+ events,
10
+ rows,
11
+ async ensure() { },
12
+ async append(appended) {
13
+ for (const event of appended) {
14
+ const row = toJournalRow(event, 'memory', now());
15
+ if (rows.some((existing) => existing.event_id === row.event_id && existing.payload_hash === row.payload_hash))
16
+ continue;
17
+ events.push(event);
18
+ rows.push(row);
19
+ }
20
+ },
21
+ async readCheckpoint(namespaceId) {
22
+ // Older test fixtures restore a journal by copying its public events.
23
+ // Materialize any such events before projecting the checkpoint so the
24
+ // in-memory adapter preserves that supported testing workflow.
25
+ const known = new Set(rows.map((row) => `${row.event_id}\0${row.payload_hash}`));
26
+ for (const event of events) {
27
+ const row = toJournalRow(event, 'memory', now());
28
+ const identity = `${row.event_id}\0${row.payload_hash}`;
29
+ if (known.has(identity))
30
+ continue;
31
+ known.add(identity);
32
+ rows.push(row);
33
+ }
34
+ return validateJournalHistory(rows.filter((row) => row.namespace_id === namespaceId), namespaceId).checkpoint;
35
+ },
36
+ };
37
+ }
38
+ /** In-memory destination that honours `insert_deduplication_token` like ClickHouse. */
39
+ export function createMemoryDestination() {
40
+ const tables = new Map();
41
+ const tokens = [];
42
+ return {
43
+ tables,
44
+ tokens,
45
+ async insert({ table, rows, token }) {
46
+ const key = `${table.database}.${table.name}`;
47
+ if (tokens.includes(`${key}:${token}`))
48
+ return;
49
+ tokens.push(`${key}:${token}`);
50
+ tables.set(key, [...(tables.get(key) ?? []), ...rows]);
51
+ },
52
+ };
53
+ }
54
+ //# sourceMappingURL=testing.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"testing.js","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAA;AAC7D,OAAO,EAAE,YAAY,EAAmB,MAAM,cAAc,CAAA;AAc5D,kFAAkF;AAClF,MAAM,UAAU,mBAAmB,CAAC,UAAgC,EAAE;IACpE,MAAM,MAAM,GAAmB,EAAE,CAAA;IACjC,MAAM,IAAI,GAAiB,EAAE,CAAA;IAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;IAC7C,OAAO;QACL,MAAM;QACN,IAAI;QACJ,KAAK,CAAC,MAAM,KAAI,CAAC;QACjB,KAAK,CAAC,MAAM,CAAC,QAAQ;YACnB,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAA;gBAChD,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,YAAY,KAAK,GAAG,CAAC,YAAY,CAAC;oBAAE,SAAQ;gBACvH,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAClB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,cAAc,CAAC,WAAW;YAC9B,sEAAsE;YACtE,sEAAsE;YACtE,+DAA+D;YAC/D,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAA;YAChF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAA;gBAChD,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,YAAY,EAAE,CAAA;gBACvD,IAAI,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC;oBAAE,SAAQ;gBACjC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;gBACnB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChB,CAAC;YACD,OAAO,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,KAAK,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC,UAAU,CAAA;QAC/G,CAAC;KACF,CAAA;AACH,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,uBAAuB;IACrC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAA;IACvC,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,OAAO;QACL,MAAM;QACN,MAAM;QACN,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE;YACjC,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAA;YAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;gBAAE,OAAM;YAC9C,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAA;YAC9B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;QACxD,CAAC;KACF,CAAA;AACH,CAAC"}
@@ -0,0 +1,229 @@
1
+ import type { Options as PRetryOptions, RetryContext as PRetryContext } from 'p-retry';
2
+ import type { TableDefinition } from '@chkit/core';
3
+ import type { FetchFailure } from './errors.js';
4
+ /** JSON-compatible column values; the ClickHouse destination encodes bigint as decimal strings. */
5
+ export type Row = Record<string, unknown>;
6
+ /**
7
+ * One bounded unit produced by a stream reader. Rows are already shaped for the
8
+ * destination table: mapping is ordinary code inside the reader.
9
+ *
10
+ * `state` is optional and, when present, must be the COMPLETE candidate
11
+ * provider state that is safe to commit once every row up to and including
12
+ * this chunk has sink evidence. The executor never infers ordering: a reader
13
+ * that cannot claim a safe frontier simply omits `state`.
14
+ */
15
+ export interface SourceChunk<TRow extends Row = Row, TState = unknown> {
16
+ rows: readonly TRow[];
17
+ state?: TState;
18
+ /**
19
+ * Stable, non-secret identity of the logical source interval this chunk
20
+ * covers (a page cursor, an id range, a day…). Declare it only when a replay
21
+ * of the same interval is the same logical write: batch identity then ignores
22
+ * row content, so mutable provider fields cannot defeat retry deduplication.
23
+ * Without it, identity falls back to a content hash, which prefers a possible
24
+ * duplicate over suppressing rows that changed between attempts.
25
+ */
26
+ id?: string;
27
+ }
28
+ export interface AttemptOptions {
29
+ /** Short non-secret label used in journal and telemetry, e.g. `GET /v2/objects`. */
30
+ label?: string;
31
+ }
32
+ /** Source request capabilities, independent of selection and checkpoint types. */
33
+ export interface FetchContext {
34
+ signal: AbortSignal;
35
+ /**
36
+ * Run one source operation under executor authority: fetch permit, retry
37
+ * policy, failure classification and cancellation. Operations that bypass it
38
+ * only get coarse recovery (reader recreation from the last checkpoint).
39
+ */
40
+ attempt<T>(operation: (signal: AbortSignal) => Promise<T>, options?: AttemptOptions): Promise<T>;
41
+ }
42
+ export interface ReadContext<TSelection, TState> extends FetchContext {
43
+ streamId: string;
44
+ /** What to read this execution, planned by the incremental strategy. */
45
+ selection: TSelection;
46
+ /** Last committed provider state, already validated by the strategy. */
47
+ state: TState | undefined;
48
+ /** Immutable cutoff shared by every stream selected in this execution. */
49
+ cutoff: Date;
50
+ }
51
+ /**
52
+ * Provider-owned incremental strategy inside the ChKit-owned checkpoint
53
+ * envelope `{ strategy, version, state }`. Core treats `state` as opaque.
54
+ */
55
+ export interface IncrementalStrategy<TState, TSelection> {
56
+ readonly id: string;
57
+ readonly version: number;
58
+ /** Validate previously committed state. Throw when it cannot be trusted. */
59
+ parseState(raw: unknown): TState;
60
+ plan(input: PlanInput<TState>): TSelection;
61
+ /**
62
+ * Candidate state once the WHOLE selection has sink evidence. Return
63
+ * `undefined` to leave the checkpoint untouched.
64
+ */
65
+ complete?(input: {
66
+ state: TState | undefined;
67
+ selection: TSelection;
68
+ }): TState | undefined;
69
+ }
70
+ export interface PlanInput<TState> {
71
+ state: TState | undefined;
72
+ cutoff: Date;
73
+ /** Explicit historical range from a backfill invocation. */
74
+ range: {
75
+ from: Date | undefined;
76
+ to: Date | undefined;
77
+ } | undefined;
78
+ }
79
+ /** p-retry context with the original provider failure and its classification. */
80
+ export type RetryContext = Omit<PRetryContext, 'error'> & {
81
+ error: FetchFailure;
82
+ };
83
+ export type RetryOptions = Pick<PRetryOptions, 'retries' | 'factor' | 'minTimeout' | 'maxTimeout' | 'randomize' | 'maxRetryTime'> & {
84
+ shouldRetry?: (context: RetryContext) => boolean | Promise<boolean>;
85
+ shouldConsumeRetry?: (context: RetryContext) => boolean | Promise<boolean>;
86
+ };
87
+ export interface StreamBudget {
88
+ /** Maximum source chunks pulled in one execution of this stream. */
89
+ maxChunks?: number;
90
+ /** Largest chunk a reader may yield. A bigger chunk fails the stream instead of being buffered. */
91
+ maxChunkRows?: number;
92
+ }
93
+ export interface StreamDefinition<TRow extends Row = Row, TState = unknown, TSelection = unknown> {
94
+ readonly kind: 'ingest_stream';
95
+ /** Globally stable identity. Owns the checkpoint; never derived from the pipeline. */
96
+ readonly id: string;
97
+ readonly destination: TableDefinition;
98
+ readonly tags: readonly string[];
99
+ readonly incremental: IncrementalStrategy<TState, TSelection>;
100
+ readonly read: (context: ReadContext<TSelection, TState>) => AsyncIterable<SourceChunk<TRow, TState>>;
101
+ readonly loader: LoaderFactory | undefined;
102
+ readonly retry: RetryOptions | undefined;
103
+ readonly batchSize: number | undefined;
104
+ readonly budget: StreamBudget | undefined;
105
+ readonly classifyError: ErrorClassifier | undefined;
106
+ }
107
+ export type AnyStreamDefinition = StreamDefinition<any, any, any>;
108
+ export interface PipelineDefinition {
109
+ readonly kind: 'ingest_pipeline';
110
+ readonly id: string;
111
+ readonly tags: readonly string[];
112
+ readonly streams: readonly AnyStreamDefinition[];
113
+ readonly maxStreams: number;
114
+ readonly maxFetches: number;
115
+ readonly maxLoads: number;
116
+ readonly retry: RetryOptions | undefined;
117
+ }
118
+ export type FailureClass = {
119
+ kind: 'cancelled';
120
+ } | {
121
+ kind: 'rate_limited';
122
+ retryAfterMs: number | undefined;
123
+ } | {
124
+ kind: 'transient';
125
+ } | {
126
+ kind: 'permanent';
127
+ } | {
128
+ kind: 'unknown';
129
+ };
130
+ export type ErrorClassifier = (cause: unknown, fallback: FailureClass) => FailureClass | undefined;
131
+ export type SinkEvidenceKind = 'clickhouse_ack' | 'none_required';
132
+ export interface SinkReceipt {
133
+ evidence: SinkEvidenceKind;
134
+ rows: number;
135
+ writeUnits: number;
136
+ }
137
+ export interface LoadBatch {
138
+ /** Stable across every retry and crash replay of the same logical batch. */
139
+ batchId: string;
140
+ rows: readonly Row[];
141
+ }
142
+ export interface LoaderContext {
143
+ streamId: string;
144
+ runId: string;
145
+ table: TableDefinition;
146
+ destination: DestinationAdapter;
147
+ signal: AbortSignal;
148
+ }
149
+ export interface Loader {
150
+ readonly ctx: LoaderContext;
151
+ write(batch: LoadBatch): Promise<void>;
152
+ finalize(): Promise<SinkReceipt>;
153
+ abort(reason: unknown): Promise<void>;
154
+ }
155
+ export type LoaderFactory = (ctx: LoaderContext) => Loader;
156
+ /** Low-level destination writes performed on behalf of a portable Loader. */
157
+ export interface DestinationAdapter {
158
+ insert(input: {
159
+ table: TableDefinition;
160
+ rows: readonly Row[];
161
+ /** Deterministic `insert_deduplication_token` for this write unit. */
162
+ token: string;
163
+ }): Promise<void>;
164
+ }
165
+ export type JournalEventKind = 'run_started' | 'work_planned' | 'attempt_started' | 'retry_scheduled' | 'batch_committed' | 'work_finished' | 'run_finished';
166
+ export type WorkState = '' | 'planned' | 'running' | 'succeeded' | 'failed' | 'budget_exhausted' | 'cancelled';
167
+ export interface CheckpointEnvelope {
168
+ strategy: string;
169
+ version: number;
170
+ state: unknown;
171
+ }
172
+ export interface JournalEvent {
173
+ namespaceId: string;
174
+ eventSeq: number;
175
+ eventKind: JournalEventKind;
176
+ runId: string;
177
+ workId: string;
178
+ attemptNo: number;
179
+ batchId: string;
180
+ expectedCheckpointVersion: number;
181
+ checkpointVersion: number;
182
+ checkpoint: CheckpointEnvelope | undefined;
183
+ workState: WorkState;
184
+ sinkEvidence: SinkEvidenceKind | '';
185
+ retryAt: Date | undefined;
186
+ errorClass: string;
187
+ detail: Record<string, unknown>;
188
+ }
189
+ export interface CommittedCheckpoint {
190
+ version: number;
191
+ envelope: CheckpointEnvelope | undefined;
192
+ /** Identity of the last state advancement; empty for an initial checkpoint. */
193
+ checkpointId: string;
194
+ /** Identity of the last successful sync; separates new syncs from crash replay. */
195
+ successId: string;
196
+ /** Last sequence observed within the selected run, not a shared allocator. */
197
+ headSeq: number;
198
+ /** Local sequence of the last successful work_finished, or zero. */
199
+ lastSuccessSeq: number;
200
+ }
201
+ /** Append-only run histories; checkpoints are projections of acknowledged work. */
202
+ export interface Journal {
203
+ ensure(): Promise<void>;
204
+ /**
205
+ * Facts written together share one insert. Retry an ambiguous failure with
206
+ * the exact same run-scoped identities and payloads.
207
+ */
208
+ append(events: readonly JournalEvent[]): Promise<void>;
209
+ readCheckpoint(namespaceId: string): Promise<CommittedCheckpoint>;
210
+ }
211
+ export type StreamOutcome = 'succeeded' | 'failed' | 'budget_exhausted' | 'cancelled';
212
+ export interface StreamResult {
213
+ streamId: string;
214
+ pipelineId: string;
215
+ namespaceId: string;
216
+ outcome: StreamOutcome;
217
+ rows: number;
218
+ batches: number;
219
+ chunks: number;
220
+ checkpointVersion: number;
221
+ error: string | undefined;
222
+ }
223
+ export interface ExecutionResult {
224
+ runId: string;
225
+ cutoff: string;
226
+ streams: StreamResult[];
227
+ ok: boolean;
228
+ }
229
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,IAAI,aAAa,EAAE,YAAY,IAAI,aAAa,EAAE,MAAM,SAAS,CAAA;AAEtF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAElD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAI/C,mGAAmG;AACnG,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;AAEzC;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW,CAAC,IAAI,SAAS,GAAG,GAAG,GAAG,EAAE,MAAM,GAAG,OAAO;IACnE,IAAI,EAAE,SAAS,IAAI,EAAE,CAAA;IACrB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;;;;;OAOG;IACH,EAAE,CAAC,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,cAAc;IAC7B,oFAAoF;IACpF,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,kFAAkF;AAClF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,WAAW,CAAA;IACnB;;;;OAIG;IACH,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;CACjG;AAED,MAAM,WAAW,WAAW,CAAC,UAAU,EAAE,MAAM,CAAE,SAAQ,YAAY;IACnE,QAAQ,EAAE,MAAM,CAAA;IAChB,wEAAwE;IACxE,SAAS,EAAE,UAAU,CAAA;IACrB,wEAAwE;IACxE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAA;IACzB,0EAA0E;IAC1E,MAAM,EAAE,IAAI,CAAA;CACb;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB,CAAC,MAAM,EAAE,UAAU;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,4EAA4E;IAC5E,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAAA;IAChC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,UAAU,CAAA;IAC1C;;;OAGG;IACH,QAAQ,CAAC,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;QAAC,SAAS,EAAE,UAAU,CAAA;KAAE,GAAG,MAAM,GAAG,SAAS,CAAA;CAC3F;AAED,MAAM,WAAW,SAAS,CAAC,MAAM;IAC/B,KAAK,EAAE,MAAM,GAAG,SAAS,CAAA;IACzB,MAAM,EAAE,IAAI,CAAA;IACZ,4DAA4D;IAC5D,KAAK,EAAE;QAAE,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;QAAC,EAAE,EAAE,IAAI,GAAG,SAAS,CAAA;KAAE,GAAG,SAAS,CAAA;CACpE;AAED,iFAAiF;AACjF,MAAM,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,GAAG;IAAE,KAAK,EAAE,YAAY,CAAA;CAAE,CAAA;AAEjF,MAAM,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAC3C,SAAS,GAAG,QAAQ,GAAG,YAAY,GAAG,YAAY,GAAG,WAAW,GAAG,cAAc,CAClF,GAAG;IACF,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IACnE,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;CAC3E,CAAA;AAED,MAAM,WAAW,YAAY;IAC3B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,mGAAmG;IACnG,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,gBAAgB,CAAC,IAAI,SAAS,GAAG,GAAG,GAAG,EAAE,MAAM,GAAG,OAAO,EAAE,UAAU,GAAG,OAAO;IAC9F,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAA;IAC9B,sFAAsF;IACtF,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,WAAW,EAAE,eAAe,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IAChC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC7D,QAAQ,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,aAAa,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;IACrG,QAAQ,CAAC,MAAM,EAAE,aAAa,GAAG,SAAS,CAAA;IAC1C,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,CAAA;IACzC,QAAQ,CAAC,aAAa,EAAE,eAAe,GAAG,SAAS,CAAA;CACpD;AAGD,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAEjE,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAA;IAChC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IAChC,QAAQ,CAAC,OAAO,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,SAAS,CAAA;CACzC;AAID,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,GAC1D;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,CAAA;AAEvB,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,KAAK,YAAY,GAAG,SAAS,CAAA;AAIlG,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,GAAG,eAAe,CAAA;AAEjE,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,SAAS;IACxB,4EAA4E;IAC5E,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,SAAS,GAAG,EAAE,CAAA;CACrB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,eAAe,CAAA;IACtB,WAAW,EAAE,kBAAkB,CAAA;IAC/B,MAAM,EAAE,WAAW,CAAA;CACpB;AAED,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,GAAG,EAAE,aAAa,CAAA;IAC3B,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACtC,QAAQ,IAAI,OAAO,CAAC,WAAW,CAAC,CAAA;IAChC,KAAK,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACtC;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,GAAG,EAAE,aAAa,KAAK,MAAM,CAAA;AAE1D,6EAA6E;AAC7E,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,KAAK,EAAE;QACZ,KAAK,EAAE,eAAe,CAAA;QACtB,IAAI,EAAE,SAAS,GAAG,EAAE,CAAA;QACpB,sEAAsE;QACtE,KAAK,EAAE,MAAM,CAAA;KACd,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAClB;AAID,MAAM,MAAM,gBAAgB,GACxB,aAAa,GACb,cAAc,GACd,iBAAiB,GACjB,iBAAiB,GACjB,iBAAiB,GACjB,eAAe,GACf,cAAc,CAAA;AAElB,MAAM,MAAM,SAAS,GAAG,EAAE,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,kBAAkB,GAAG,WAAW,CAAA;AAE9G,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,OAAO,CAAA;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,gBAAgB,CAAA;IAC3B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,yBAAyB,EAAE,MAAM,CAAA;IACjC,iBAAiB,EAAE,MAAM,CAAA;IACzB,UAAU,EAAE,kBAAkB,GAAG,SAAS,CAAA;IAC1C,SAAS,EAAE,SAAS,CAAA;IACpB,YAAY,EAAE,gBAAgB,GAAG,EAAE,CAAA;IACnC,OAAO,EAAE,IAAI,GAAG,SAAS,CAAA;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAChC;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,kBAAkB,GAAG,SAAS,CAAA;IACxC,+EAA+E;IAC/E,YAAY,EAAE,MAAM,CAAA;IACpB,mFAAmF;IACnF,SAAS,EAAE,MAAM,CAAA;IACjB,8EAA8E;IAC9E,OAAO,EAAE,MAAM,CAAA;IACf,oEAAoE;IACpE,cAAc,EAAE,MAAM,CAAA;CACvB;AAED,mFAAmF;AACnF,MAAM,WAAW,OAAO;IACtB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACvB;;;OAGG;IACH,MAAM,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACtD,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAA;CAClE;AAID,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,kBAAkB,GAAG,WAAW,CAAA;AAErF,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,aAAa,CAAA;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,iBAAiB,EAAE,MAAM,CAAA;IACzB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAA;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,EAAE,EAAE,OAAO,CAAA;CACZ"}
package/dist/types.js ADDED
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}