@sequenceholdings/pipeline-spec 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -0
- package/dist/authoring/per-table-ingestion.d.ts +74 -0
- package/dist/authoring/per-table-ingestion.d.ts.map +1 -0
- package/dist/authoring/per-table-ingestion.js +122 -0
- package/dist/authoring/per-table-ingestion.js.map +1 -0
- package/dist/authoring/yaml.d.ts +6 -0
- package/dist/authoring/yaml.d.ts.map +1 -0
- package/dist/authoring/yaml.js +15 -0
- package/dist/authoring/yaml.js.map +1 -0
- package/dist/binding-gaps.d.ts +21 -0
- package/dist/binding-gaps.d.ts.map +1 -0
- package/dist/binding-gaps.js +74 -0
- package/dist/binding-gaps.js.map +1 -0
- package/dist/bodies/ingestion.d.ts +359 -0
- package/dist/bodies/ingestion.d.ts.map +1 -0
- package/dist/bodies/ingestion.js +446 -0
- package/dist/bodies/ingestion.js.map +1 -0
- package/dist/bodies/serving.d.ts +316 -0
- package/dist/bodies/serving.d.ts.map +1 -0
- package/dist/bodies/serving.js +290 -0
- package/dist/bodies/serving.js.map +1 -0
- package/dist/bodies/transformation.d.ts +397 -0
- package/dist/bodies/transformation.d.ts.map +1 -0
- package/dist/bodies/transformation.js +278 -0
- package/dist/bodies/transformation.js.map +1 -0
- package/dist/catalog-binding.d.ts +15 -0
- package/dist/catalog-binding.d.ts.map +1 -0
- package/dist/catalog-binding.js +15 -0
- package/dist/catalog-binding.js.map +1 -0
- package/dist/columns.d.ts +177 -0
- package/dist/columns.d.ts.map +1 -0
- package/dist/columns.js +89 -0
- package/dist/columns.js.map +1 -0
- package/dist/compatibility.d.ts +25 -0
- package/dist/compatibility.d.ts.map +1 -0
- package/dist/compatibility.js +108 -0
- package/dist/compatibility.js.map +1 -0
- package/dist/constants.d.ts +95 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +105 -0
- package/dist/constants.js.map +1 -0
- package/dist/diff.d.ts +36 -0
- package/dist/diff.d.ts.map +1 -0
- package/dist/diff.js +300 -0
- package/dist/diff.js.map +1 -0
- package/dist/documentation.d.ts +22 -0
- package/dist/documentation.d.ts.map +1 -0
- package/dist/documentation.js +43 -0
- package/dist/documentation.js.map +1 -0
- package/dist/envelope.d.ts +1241 -0
- package/dist/envelope.d.ts.map +1 -0
- package/dist/envelope.js +163 -0
- package/dist/envelope.js.map +1 -0
- package/dist/errors.d.ts +30 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +47 -0
- package/dist/errors.js.map +1 -0
- package/dist/graph-references.d.ts +34 -0
- package/dist/graph-references.d.ts.map +1 -0
- package/dist/graph-references.js +81 -0
- package/dist/graph-references.js.map +1 -0
- package/dist/graph.d.ts +112 -0
- package/dist/graph.d.ts.map +1 -0
- package/dist/graph.js +546 -0
- package/dist/graph.js.map +1 -0
- package/dist/hash.d.ts +28 -0
- package/dist/hash.d.ts.map +1 -0
- package/dist/hash.js +91 -0
- package/dist/hash.js.map +1 -0
- package/dist/index.d.ts +27 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +27 -0
- package/dist/index.js.map +1 -0
- package/dist/issues.d.ts +14 -0
- package/dist/issues.d.ts.map +1 -0
- package/dist/issues.js +27 -0
- package/dist/issues.js.map +1 -0
- package/dist/loader.d.ts +50 -0
- package/dist/loader.d.ts.map +1 -0
- package/dist/loader.js +241 -0
- package/dist/loader.js.map +1 -0
- package/dist/migrations/contracts.d.ts +38 -0
- package/dist/migrations/contracts.d.ts.map +1 -0
- package/dist/migrations/contracts.js +175 -0
- package/dist/migrations/contracts.js.map +1 -0
- package/dist/migrations/manifest.d.ts +137 -0
- package/dist/migrations/manifest.d.ts.map +1 -0
- package/dist/migrations/manifest.js +137 -0
- package/dist/migrations/manifest.js.map +1 -0
- package/dist/operations.d.ts +214 -0
- package/dist/operations.d.ts.map +1 -0
- package/dist/operations.js +275 -0
- package/dist/operations.js.map +1 -0
- package/dist/orm-satisfies.d.ts +132 -0
- package/dist/orm-satisfies.d.ts.map +1 -0
- package/dist/orm-satisfies.js +119 -0
- package/dist/orm-satisfies.js.map +1 -0
- package/dist/primitives.d.ts +11 -0
- package/dist/primitives.d.ts.map +1 -0
- package/dist/primitives.js +20 -0
- package/dist/primitives.js.map +1 -0
- package/dist/resolve.d.ts +124 -0
- package/dist/resolve.d.ts.map +1 -0
- package/dist/resolve.js +194 -0
- package/dist/resolve.js.map +1 -0
- package/dist/secret-scan.d.ts +34 -0
- package/dist/secret-scan.d.ts.map +1 -0
- package/dist/secret-scan.js +98 -0
- package/dist/secret-scan.js.map +1 -0
- package/dist/serving-compatibility.d.ts +14 -0
- package/dist/serving-compatibility.d.ts.map +1 -0
- package/dist/serving-compatibility.js +19 -0
- package/dist/serving-compatibility.js.map +1 -0
- package/dist/snapshot-transport.d.ts +83 -0
- package/dist/snapshot-transport.d.ts.map +1 -0
- package/dist/snapshot-transport.js +42 -0
- package/dist/snapshot-transport.js.map +1 -0
- package/dist/suggestions.d.ts +3 -0
- package/dist/suggestions.d.ts.map +1 -0
- package/dist/suggestions.js +26 -0
- package/dist/suggestions.js.map +1 -0
- package/dist/types.d.ts +72 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +119 -0
- package/dist/types.js.map +1 -0
- package/package.json +52 -0
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { isoDurationSchema, repoRelativePathSchema, symbolicRefSchema } from '../primitives.js';
|
|
3
|
+
/**
|
|
4
|
+
* Ingestion stages declare **feeds into a volume landing zone** — never bronze
|
|
5
|
+
* tables, which are platform-derived. This body encodes the append-only-bronze
|
|
6
|
+
* architecture: volumes-first landing, snapshot_mode batch semantics, symbolic
|
|
7
|
+
* worker/credential bindings, and the decode/write split.
|
|
8
|
+
*
|
|
9
|
+
* ## What the platform derives from a feed (not authored)
|
|
10
|
+
*
|
|
11
|
+
* Ingestion lands in **catalog-per-source** catalogs (estate precedent:
|
|
12
|
+
* `src_banksouth_jack_henry`, `src_mock_data`): bronze lives in
|
|
13
|
+
* `src_<source>`, an ACCOUNT-LEVEL asset shared across deployments with a
|
|
14
|
+
* single writing deployment. The deployment catalog (`TargetFacts.catalog`)
|
|
15
|
+
* owns silver/gold/report/serving ONLY and never appears in an ingestion name.
|
|
16
|
+
*
|
|
17
|
+
* - **Source catalog** — `src_<source>` (canonical derivation:
|
|
18
|
+
* `sourceCatalog`; the source identity is underscore-constrained, so the
|
|
19
|
+
* catalog name is deterministic and UC-legal).
|
|
20
|
+
* - **Volume layout** — `/Volumes/src_<source>/bronze/<feed>/batch_id=<ts>/`
|
|
21
|
+
* (UC volumes are `/Volumes/<catalog>/<schema>/<volume>/…`, so the schema
|
|
22
|
+
* is `bronze` and the volume is the feed): every batch lands as raw files
|
|
23
|
+
* under a batch directory keyed by ingestion timestamp. Canonical
|
|
24
|
+
* derivation: `feedVolumePath` (tests pin the documented string to it).
|
|
25
|
+
* - **Bronze table name** — one append-only bronze table per feed:
|
|
26
|
+
* `src_<source>.bronze.<feed>` — the catalog scopes the source, so the
|
|
27
|
+
* table name needs no `<source>_` prefix (canonical derivation:
|
|
28
|
+
* `feedBronzeTable`).
|
|
29
|
+
* - **Envelope columns** — the materializer stamps `batch_id`, `ingested_at`,
|
|
30
|
+
* `source_system`, and `content_hash` onto every row. These names are
|
|
31
|
+
* RESERVED (`RESERVED_ENVELOPE_COLUMNS` in `../constants.js`) — a feed's
|
|
32
|
+
* `natural_key`/`cursor.column` may not name one (`refineFeed` below);
|
|
33
|
+
* the materializer itself rejects a source column collision at runtime,
|
|
34
|
+
* since a feed's full column set is only known at decode time.
|
|
35
|
+
* - **Presence manifest** — for `full_snapshot` feeds the platform records
|
|
36
|
+
* which natural keys appeared in each batch, so downstream deletion
|
|
37
|
+
* detection never needs source-side tombstones.
|
|
38
|
+
*
|
|
39
|
+
* ## The decode/write split
|
|
40
|
+
*
|
|
41
|
+
* Decoders are developer-provided **pure functions (file contents → rows)**
|
|
42
|
+
* invoked by the platform materializer for non-standard formats. Decoders
|
|
43
|
+
* never hold table-write credentials — the platform materializer performs
|
|
44
|
+
* envelope stamping and the append. **Bronze materialization does no semantic
|
|
45
|
+
* work**: no type coercion, dedup, normalization, or filtering — that is
|
|
46
|
+
* cleaning-transformation territory.
|
|
47
|
+
*/
|
|
48
|
+
export { isoDurationSchema, repoRelativePathSchema, symbolicRefSchema };
|
|
49
|
+
export declare const SNAPSHOT_MODES: readonly ["full_snapshot", "incremental", "event_stream"];
|
|
50
|
+
export declare const snapshotModeSchema: z.ZodEnum<{
|
|
51
|
+
full_snapshot: "full_snapshot";
|
|
52
|
+
incremental: "incremental";
|
|
53
|
+
event_stream: "event_stream";
|
|
54
|
+
}>;
|
|
55
|
+
export type SnapshotMode = z.infer<typeof snapshotModeSchema>;
|
|
56
|
+
/**
|
|
57
|
+
* Feed formats. `xlsx` is RESERVED: present in the enum but rejected at
|
|
58
|
+
* validation until the platform ships its default spreadsheet decoder (sheet
|
|
59
|
+
* selection / header detection stay byte-shape decoding — type coercion stays
|
|
60
|
+
* in cleaning). `custom` requires a `decoder`.
|
|
61
|
+
*/
|
|
62
|
+
export declare const FEED_FORMATS: readonly ["parquet", "json", "csv", "xlsx", "custom"];
|
|
63
|
+
export declare const feedFormatSchema: z.ZodEnum<{
|
|
64
|
+
json: "json";
|
|
65
|
+
custom: "custom";
|
|
66
|
+
parquet: "parquet";
|
|
67
|
+
csv: "csv";
|
|
68
|
+
xlsx: "xlsx";
|
|
69
|
+
}>;
|
|
70
|
+
export type FeedFormat = z.infer<typeof feedFormatSchema>;
|
|
71
|
+
/**
|
|
72
|
+
* Incremental cursor declaration. `encoding` is an extensible string (e.g.
|
|
73
|
+
* `yyyyddd` for Jack-Henry-style Julian dates) — the SDK stays
|
|
74
|
+
* provider-neutral and never enumerates vendor encodings.
|
|
75
|
+
*/
|
|
76
|
+
export declare const cursorSchema: z.ZodObject<{
|
|
77
|
+
column: z.ZodString;
|
|
78
|
+
encoding: z.ZodOptional<z.ZodString>;
|
|
79
|
+
}, z.core.$strict>;
|
|
80
|
+
export declare const feedSchema: z.ZodObject<{
|
|
81
|
+
columns: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
82
|
+
description: z.ZodString;
|
|
83
|
+
}, z.core.$strict>>>;
|
|
84
|
+
documentation_ref: z.ZodOptional<z.ZodString>;
|
|
85
|
+
snapshot_mode: z.ZodEnum<{
|
|
86
|
+
full_snapshot: "full_snapshot";
|
|
87
|
+
incremental: "incremental";
|
|
88
|
+
event_stream: "event_stream";
|
|
89
|
+
}>;
|
|
90
|
+
natural_key: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
91
|
+
cursor: z.ZodOptional<z.ZodObject<{
|
|
92
|
+
column: z.ZodString;
|
|
93
|
+
encoding: z.ZodOptional<z.ZodString>;
|
|
94
|
+
}, z.core.$strict>>;
|
|
95
|
+
pii: z.ZodDefault<z.ZodEnum<{
|
|
96
|
+
none: "none";
|
|
97
|
+
contains_pii: "contains_pii";
|
|
98
|
+
}>>;
|
|
99
|
+
format: z.ZodOptional<z.ZodEnum<{
|
|
100
|
+
json: "json";
|
|
101
|
+
custom: "custom";
|
|
102
|
+
parquet: "parquet";
|
|
103
|
+
csv: "csv";
|
|
104
|
+
xlsx: "xlsx";
|
|
105
|
+
}>>;
|
|
106
|
+
decoder: z.ZodOptional<z.ZodString>;
|
|
107
|
+
extraction: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
|
|
108
|
+
freshness_slo: z.ZodOptional<z.ZodString>;
|
|
109
|
+
optional: z.ZodOptional<z.ZodBoolean>;
|
|
110
|
+
description: z.ZodOptional<z.ZodString>;
|
|
111
|
+
column_descriptions: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
112
|
+
}, z.core.$strict>;
|
|
113
|
+
export type FeedSpec = z.infer<typeof feedSchema>;
|
|
114
|
+
/** Feed documentation changes do not change downstream compatibility. */
|
|
115
|
+
export declare function feedContract(feed: FeedSpec): Omit<FeedSpec, 'description' | 'columns' | 'column_descriptions' | 'documentation_ref'>;
|
|
116
|
+
/**
|
|
117
|
+
* Ingestion schedule: cron + tz + the shared operation policy vocabulary.
|
|
118
|
+
* Optional by design — the future `upload` runtime is event-driven (file
|
|
119
|
+
* arrival), so a schedule is meaningless there.
|
|
120
|
+
*/
|
|
121
|
+
export declare const scheduleSchema: z.ZodObject<{
|
|
122
|
+
cron: z.ZodString;
|
|
123
|
+
tz: z.ZodString;
|
|
124
|
+
policy: z.ZodOptional<z.ZodObject<{
|
|
125
|
+
singleton: z.ZodOptional<z.ZodBoolean>;
|
|
126
|
+
exclusion_group: z.ZodOptional<z.ZodString>;
|
|
127
|
+
timeout_minutes: z.ZodOptional<z.ZodNumber>;
|
|
128
|
+
mutates: z.ZodOptional<z.ZodBoolean>;
|
|
129
|
+
cooldown_minutes: z.ZodOptional<z.ZodNumber>;
|
|
130
|
+
requires_approval: z.ZodOptional<z.ZodBoolean>;
|
|
131
|
+
}, z.core.$strict>>;
|
|
132
|
+
}, z.core.$strict>;
|
|
133
|
+
export type ScheduleSpec = z.infer<typeof scheduleSchema>;
|
|
134
|
+
/**
|
|
135
|
+
* Where the ingestion code runs:
|
|
136
|
+
* - `databricks` — a platform-scheduled job in the deployment workspace,
|
|
137
|
+
* pulling from an external system's API (`source` required).
|
|
138
|
+
* - `edge-worker` — an on-prem/VM worker registered in the platform registry
|
|
139
|
+
* (`worker` required): the SEQUENCE2-style extraction posture.
|
|
140
|
+
* - `upload` — RESERVED, rejected at validation ('not yet supported').
|
|
141
|
+
* Future semantics: no entrypoint/schedule/credential; the platform upload
|
|
142
|
+
* surface writes the feed's batch directories; the materializer fires on
|
|
143
|
+
* file arrival (Databricks file-arrival trigger); the batch ledger records
|
|
144
|
+
* the uploading initiator. Covers the spreadsheet→volume→cleaning→ontology
|
|
145
|
+
* expansion (seeds CSVs, iSeries dumps, BSM mock data).
|
|
146
|
+
*/
|
|
147
|
+
export declare const INGESTION_RUNTIMES: readonly ["databricks", "edge-worker", "upload"];
|
|
148
|
+
export declare const ingestionRuntimeSchema: z.ZodEnum<{
|
|
149
|
+
databricks: "databricks";
|
|
150
|
+
"edge-worker": "edge-worker";
|
|
151
|
+
upload: "upload";
|
|
152
|
+
}>;
|
|
153
|
+
export type IngestionRuntime = z.infer<typeof ingestionRuntimeSchema>;
|
|
154
|
+
/** Hard ceiling that keeps a malformed authoring file from stampeding a VM. */
|
|
155
|
+
export declare const MAX_WORKER_CONCURRENT = 32;
|
|
156
|
+
export declare const DEFAULT_MATERIALIZER_CONCURRENT = 32;
|
|
157
|
+
export declare const MAX_MATERIALIZER_CONCURRENT = 64;
|
|
158
|
+
export declare const MATERIALIZER_STRATEGIES: readonly ["per_feed", "sharded"];
|
|
159
|
+
export declare const materializerStrategySchema: z.ZodEnum<{
|
|
160
|
+
per_feed: "per_feed";
|
|
161
|
+
sharded: "sharded";
|
|
162
|
+
}>;
|
|
163
|
+
export type MaterializerStrategy = z.infer<typeof materializerStrategySchema>;
|
|
164
|
+
export declare const materializerConcurrencySchema: z.ZodObject<{
|
|
165
|
+
max_concurrent: z.ZodNumber;
|
|
166
|
+
strategy: z.ZodOptional<z.ZodEnum<{
|
|
167
|
+
per_feed: "per_feed";
|
|
168
|
+
sharded: "sharded";
|
|
169
|
+
}>>;
|
|
170
|
+
}, z.core.$strict>;
|
|
171
|
+
/**
|
|
172
|
+
* Symbolic worker binding — machine and Data Sync family identifiers, never
|
|
173
|
+
* connection strings or managed-secret bindings.
|
|
174
|
+
*
|
|
175
|
+
* Which fields feed physical-name derivation: `data_source` is the source
|
|
176
|
+
* identity (bronze table names + volume paths derive from it, hence the
|
|
177
|
+
* identifier grammar); `machine` is a registry machine id (SEQUENCE2-style),
|
|
178
|
+
* `credential` a Data Sync environment family, and `operation` a machine-local
|
|
179
|
+
* Data Sync operation id — none appears in any derived name.
|
|
180
|
+
*/
|
|
181
|
+
export declare const workerSchema: z.ZodObject<{
|
|
182
|
+
machine: z.ZodString;
|
|
183
|
+
data_source: z.ZodString;
|
|
184
|
+
credential: z.ZodString;
|
|
185
|
+
operation: z.ZodOptional<z.ZodString>;
|
|
186
|
+
max_concurrent: z.ZodOptional<z.ZodNumber>;
|
|
187
|
+
}, z.core.$strict>;
|
|
188
|
+
/**
|
|
189
|
+
* Symbolic source binding for databricks-runtime API pulls. `system` is the
|
|
190
|
+
* source identity (feeds derived UC names — identifier grammar).
|
|
191
|
+
*
|
|
192
|
+
* `credential` names a binding family when the upstream API is authenticated.
|
|
193
|
+
* Omit it for genuinely keyless / public sources (e.g. ESPN's public site API)
|
|
194
|
+
* — never invent a sentinel binding. Resolution and compiled `credential_*`
|
|
195
|
+
* parameters only occur when this field is present.
|
|
196
|
+
*/
|
|
197
|
+
export declare const sourceSystemSchema: z.ZodObject<{
|
|
198
|
+
system: z.ZodString;
|
|
199
|
+
credential: z.ZodOptional<z.ZodString>;
|
|
200
|
+
}, z.core.$strict>;
|
|
201
|
+
/** Feed names are snake_case identifiers (they become volume path segments and bronze table parts). */
|
|
202
|
+
export declare const feedNameSchema: z.ZodString;
|
|
203
|
+
export declare const ingestionBodyShape: {
|
|
204
|
+
runtime: z.ZodEnum<{
|
|
205
|
+
databricks: "databricks";
|
|
206
|
+
"edge-worker": "edge-worker";
|
|
207
|
+
upload: "upload";
|
|
208
|
+
}>;
|
|
209
|
+
/** Required iff `runtime: edge-worker`; forbidden otherwise. */
|
|
210
|
+
worker: z.ZodOptional<z.ZodObject<{
|
|
211
|
+
machine: z.ZodString;
|
|
212
|
+
data_source: z.ZodString;
|
|
213
|
+
credential: z.ZodString;
|
|
214
|
+
operation: z.ZodOptional<z.ZodString>;
|
|
215
|
+
max_concurrent: z.ZodOptional<z.ZodNumber>;
|
|
216
|
+
}, z.core.$strict>>;
|
|
217
|
+
/** Required iff `runtime: databricks`; forbidden otherwise. */
|
|
218
|
+
source: z.ZodOptional<z.ZodObject<{
|
|
219
|
+
system: z.ZodString;
|
|
220
|
+
credential: z.ZodOptional<z.ZodString>;
|
|
221
|
+
}, z.core.$strict>>;
|
|
222
|
+
/** Databricks-side feed materializer concurrency, independent of extraction. */
|
|
223
|
+
materializer: z.ZodOptional<z.ZodObject<{
|
|
224
|
+
max_concurrent: z.ZodNumber;
|
|
225
|
+
strategy: z.ZodOptional<z.ZodEnum<{
|
|
226
|
+
per_feed: "per_feed";
|
|
227
|
+
sharded: "sharded";
|
|
228
|
+
}>>;
|
|
229
|
+
}, z.core.$strict>>;
|
|
230
|
+
feeds: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
231
|
+
columns: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
232
|
+
description: z.ZodString;
|
|
233
|
+
}, z.core.$strict>>>;
|
|
234
|
+
documentation_ref: z.ZodOptional<z.ZodString>;
|
|
235
|
+
snapshot_mode: z.ZodEnum<{
|
|
236
|
+
full_snapshot: "full_snapshot";
|
|
237
|
+
incremental: "incremental";
|
|
238
|
+
event_stream: "event_stream";
|
|
239
|
+
}>;
|
|
240
|
+
natural_key: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
241
|
+
cursor: z.ZodOptional<z.ZodObject<{
|
|
242
|
+
column: z.ZodString;
|
|
243
|
+
encoding: z.ZodOptional<z.ZodString>;
|
|
244
|
+
}, z.core.$strict>>;
|
|
245
|
+
pii: z.ZodDefault<z.ZodEnum<{
|
|
246
|
+
none: "none";
|
|
247
|
+
contains_pii: "contains_pii";
|
|
248
|
+
}>>;
|
|
249
|
+
format: z.ZodOptional<z.ZodEnum<{
|
|
250
|
+
json: "json";
|
|
251
|
+
custom: "custom";
|
|
252
|
+
parquet: "parquet";
|
|
253
|
+
csv: "csv";
|
|
254
|
+
xlsx: "xlsx";
|
|
255
|
+
}>>;
|
|
256
|
+
decoder: z.ZodOptional<z.ZodString>;
|
|
257
|
+
extraction: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
|
|
258
|
+
freshness_slo: z.ZodOptional<z.ZodString>;
|
|
259
|
+
optional: z.ZodOptional<z.ZodBoolean>;
|
|
260
|
+
description: z.ZodOptional<z.ZodString>;
|
|
261
|
+
column_descriptions: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
262
|
+
}, z.core.$strict>>;
|
|
263
|
+
schedule: z.ZodOptional<z.ZodObject<{
|
|
264
|
+
cron: z.ZodString;
|
|
265
|
+
tz: z.ZodString;
|
|
266
|
+
policy: z.ZodOptional<z.ZodObject<{
|
|
267
|
+
singleton: z.ZodOptional<z.ZodBoolean>;
|
|
268
|
+
exclusion_group: z.ZodOptional<z.ZodString>;
|
|
269
|
+
timeout_minutes: z.ZodOptional<z.ZodNumber>;
|
|
270
|
+
mutates: z.ZodOptional<z.ZodBoolean>;
|
|
271
|
+
cooldown_minutes: z.ZodOptional<z.ZodNumber>;
|
|
272
|
+
requires_approval: z.ZodOptional<z.ZodBoolean>;
|
|
273
|
+
}, z.core.$strict>>;
|
|
274
|
+
}, z.core.$strict>>;
|
|
275
|
+
/** Spec-wide max staleness; feeds may override per-feed. */
|
|
276
|
+
freshness_slo: z.ZodOptional<z.ZodString>;
|
|
277
|
+
/** Raw-file retention window — an explicit replay buffer, not an archive. */
|
|
278
|
+
retention: z.ZodOptional<z.ZodString>;
|
|
279
|
+
/** Repo-relative code path. Required for databricks / edge-worker runtimes. */
|
|
280
|
+
entrypoint: z.ZodOptional<z.ZodString>;
|
|
281
|
+
};
|
|
282
|
+
/**
|
|
283
|
+
* The source identity a stage ingests from — `source.system` for databricks
|
|
284
|
+
* runtimes, `worker.data_source` for edge workers (stage id as the final
|
|
285
|
+
* fallback for future runtimes). This identity determines EVERY derived
|
|
286
|
+
* bronze name, so it is part of the contract surface (`contract_hash`) and
|
|
287
|
+
* the single input to the derivation helpers below.
|
|
288
|
+
*/
|
|
289
|
+
export declare function ingestionSourceIdentity(spec: {
|
|
290
|
+
readonly source?: {
|
|
291
|
+
readonly system: string;
|
|
292
|
+
};
|
|
293
|
+
readonly worker?: {
|
|
294
|
+
readonly data_source: string;
|
|
295
|
+
};
|
|
296
|
+
readonly stage: string;
|
|
297
|
+
}): string;
|
|
298
|
+
/**
|
|
299
|
+
* Canonical per-source catalog: `src_<source>`. Ingestion never lands in the
|
|
300
|
+
* deployment catalog — per-source catalogs are account-level assets shared
|
|
301
|
+
* across deployments (one writing deployment), per the locked topology.
|
|
302
|
+
*
|
|
303
|
+
* Because the catalog is ACCOUNT-level, the source identity is an
|
|
304
|
+
* account-global namespace: where a source system spans tenants, the author
|
|
305
|
+
* scopes the tenant INTO the identity (`banksouth_candescent`, matching the
|
|
306
|
+
* estate's `src_banksouth_jack_henry`); first-party/global sources need no
|
|
307
|
+
* scope (`src_mock_data`). The registry milestone enforces single-writer per
|
|
308
|
+
* `src_*` FQN account-wide and may introduce registry-owned catalog bindings
|
|
309
|
+
* that supersede this deterministic default.
|
|
310
|
+
*/
|
|
311
|
+
export declare function sourceCatalog(source: string): string;
|
|
312
|
+
/**
|
|
313
|
+
* Canonical volume layout for a feed's batches:
|
|
314
|
+
* `/Volumes/src_<source>/bronze/<feed>` — batches land under
|
|
315
|
+
* `batch_id=<ts>/` inside it. The ONE derivation the resolver, docs, and
|
|
316
|
+
* tests share. Derived from the spec's own source identity — never from
|
|
317
|
+
* `TargetFacts.catalog`.
|
|
318
|
+
*/
|
|
319
|
+
export declare function feedVolumePath(args: {
|
|
320
|
+
source: string;
|
|
321
|
+
feed: string;
|
|
322
|
+
}): string;
|
|
323
|
+
/** Canonical bronze table FQN for a feed: `src_<source>.bronze.<feed>`. */
|
|
324
|
+
export declare function feedBronzeTable(args: {
|
|
325
|
+
source: string;
|
|
326
|
+
feed: string;
|
|
327
|
+
}): string;
|
|
328
|
+
/**
|
|
329
|
+
* Canonical capacity/exclusion group for every stage extracting from ONE
|
|
330
|
+
* machine + data source: `<machine>_<data_source>`.
|
|
331
|
+
*
|
|
332
|
+
* This is the anti-stampede identity. With one stage per table, ~150 sibling
|
|
333
|
+
* stages point at a single VM holding a single ODBC connection to the core; a
|
|
334
|
+
* per-stage exclusion group would declare them all mutually independent and let
|
|
335
|
+
* every one of them fire at once. Grouping by the contended upstream instead
|
|
336
|
+
* means the declared policy matches physical reality, and `worker.max_concurrent`
|
|
337
|
+
* is the budget within the group.
|
|
338
|
+
*
|
|
339
|
+
* Machine ids permit hyphens (symbolic-ref grammar) while group names are
|
|
340
|
+
* identifiers, so hyphens are normalized to underscores.
|
|
341
|
+
*/
|
|
342
|
+
export declare function workerCapacityGroup(worker: {
|
|
343
|
+
readonly machine: string;
|
|
344
|
+
readonly data_source: string;
|
|
345
|
+
}): string;
|
|
346
|
+
/** Structural slice of a stage spec the ingestion refinement needs. */
|
|
347
|
+
export interface IngestionBody {
|
|
348
|
+
readonly runtime: IngestionRuntime;
|
|
349
|
+
readonly worker?: z.infer<typeof workerSchema>;
|
|
350
|
+
readonly source?: z.infer<typeof sourceSystemSchema>;
|
|
351
|
+
readonly feeds: Record<string, FeedSpec>;
|
|
352
|
+
readonly entrypoint?: string;
|
|
353
|
+
readonly slos?: {
|
|
354
|
+
readonly freshness?: string;
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
/** Cross-field validation for `type: ingestion` bodies (invoked from the envelope refinement). */
|
|
358
|
+
export declare function refineIngestionBody(body: IngestionBody, ctx: z.RefinementCtx): void;
|
|
359
|
+
//# sourceMappingURL=ingestion.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingestion.d.ts","sourceRoot":"","sources":["../../src/bodies/ingestion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAUvB,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAE/F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAIH,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,CAAA;AAEvE,eAAO,MAAM,cAAc,2DAA4D,CAAA;AACvF,eAAO,MAAM,kBAAkB;;;;EAAyB,CAAA;AACxD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAA;AAE7D;;;;;GAKG;AACH,eAAO,MAAM,YAAY,uDAAwD,CAAA;AACjF,eAAO,MAAM,gBAAgB;;;;;;EAAuB,CAAA;AACpD,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAA;AAEzD;;;;GAIG;AACH,eAAO,MAAM,YAAY;;;kBAOvB,CAAA;AAKF,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuCrB,CAAA;AAEF,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA;AAEjD,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,IAAI,EAAE,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,GAAG,SAAS,GAAG,qBAAqB,GAAG,mBAAmB,CAAC,CAGpI;AAED;;;;GAIG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;kBAIzB,CAAA;AAEF,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAA;AAEzD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,kBAAkB,kDAAmD,CAAA;AAClF,eAAO,MAAM,sBAAsB;;;;EAA6B,CAAA;AAChE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAA;AAerE,+EAA+E;AAC/E,eAAO,MAAM,qBAAqB,KAAK,CAAA;AACvC,eAAO,MAAM,+BAA+B,KAAK,CAAA;AACjD,eAAO,MAAM,2BAA2B,KAAK,CAAA;AAE7C,eAAO,MAAM,uBAAuB,kCAAmC,CAAA;AACvE,eAAO,MAAM,0BAA0B;;;EAAkC,CAAA;AACzE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAA;AAE7E,eAAO,MAAM,6BAA6B;;;;;;kBAmBxC,CAAA;AAEF;;;;;;;;;GASG;AACH,eAAO,MAAM,YAAY;;;;;;kBAwBvB,CAAA;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB;;;kBAO7B,CAAA;AAEF,uGAAuG;AACvG,eAAO,MAAM,cAAc,aAE2D,CAAA;AAEtF,eAAO,MAAM,kBAAkB;;;;;;IAE7B,gEAAgE;;;;;;;;IAEhE,+DAA+D;;;;;IAE/D,gFAAgF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAIhF,4DAA4D;;IAE5D,6EAA6E;;IAE7E,+EAA+E;;CAExD,CAAA;AAEzB;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE;IAC5C,QAAQ,CAAC,MAAM,CAAC,EAAE;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;IAC7C,QAAQ,CAAC,MAAM,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAA;IAClD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB,GAAG,MAAM,CAET;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAE7E;AAED,2EAA2E;AAC3E,wBAAgB,eAAe,CAAC,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAE9E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE;IAC1C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B,GAAG,MAAM,CAET;AAED,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAA;IAClC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAA;IAC9C,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAA;IACpD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IACxC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAChD;AAED,kGAAkG;AAClG,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,aAAa,GAAG,IAAI,CAwEnF"}
|