@dudousxd/nestjs-catalog 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.environment.d.ts +12 -1
- package/dist/catalog.environment.js +12 -1
- package/dist/catalog.pipeline.d.ts +548 -3
- package/dist/catalog.pipeline.js +570 -1
- package/dist/catalog.stage-encoding.d.ts +72 -0
- package/dist/catalog.stage-encoding.js +100 -0
- package/dist/client.d.ts +3 -3
- package/dist/client.js +31 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +21 -4
- package/dist/transform-runner.d.ts +42 -1
- package/dist/transform-runner.js +677 -28
- package/package.json +1 -1
|
@@ -183,3 +183,75 @@ export declare function classifyStagePayload(stored: unknown): StagePayload;
|
|
|
183
183
|
* here instead of falling through to a silent `[]`.
|
|
184
184
|
*/
|
|
185
185
|
export declare function decodeStageRows(stored: unknown): Array<Record<string, unknown>>;
|
|
186
|
+
/**
|
|
187
|
+
* A rename, reduced to what the encoding needs to know.
|
|
188
|
+
*
|
|
189
|
+
* A `Map` rather than the node's `Record`, because this runs once per shape and
|
|
190
|
+
* a `Map` lookup is what the loop below wants; and separate from
|
|
191
|
+
* `WorkflowRenameNode` so this file keeps knowing nothing about graphs.
|
|
192
|
+
*/
|
|
193
|
+
export interface StageRenamePlan {
|
|
194
|
+
/** Old name → new name, applied simultaneously. */
|
|
195
|
+
readonly columns: ReadonlyMap<string, string>;
|
|
196
|
+
/** Whether a column the plan does not name survives. See the node's docblock. */
|
|
197
|
+
readonly dropUnnamed: boolean;
|
|
198
|
+
}
|
|
199
|
+
/** What {@link renameStagePayload} did, in enough detail for a run to report it. */
|
|
200
|
+
export interface StageRenameResult {
|
|
201
|
+
/** Ready to store, in the same column the batch came out of. */
|
|
202
|
+
readonly payload: ColumnarStageBatch;
|
|
203
|
+
readonly rows: number;
|
|
204
|
+
/**
|
|
205
|
+
* Whether **no value moved** — the batch came back columnar and only its
|
|
206
|
+
* `shapes` were rewritten.
|
|
207
|
+
*
|
|
208
|
+
* The claim the rename node is built on, reported rather than assumed: it is
|
|
209
|
+
* true for a pure rename over a columnar batch and false the moment unnamed
|
|
210
|
+
* columns are dropped, because dropping one removes a position from every
|
|
211
|
+
* `values` row. A run that says which one happened is a run whose cost can be
|
|
212
|
+
* explained afterwards.
|
|
213
|
+
*/
|
|
214
|
+
readonly metadataOnly: boolean;
|
|
215
|
+
/** How many entries of `shapes` came out different from what went in. */
|
|
216
|
+
readonly shapesRewritten: number;
|
|
217
|
+
/** The plan's source columns that were present in at least one shape. */
|
|
218
|
+
readonly matched: ReadonlySet<string>;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Rename the columns of one staged batch.
|
|
222
|
+
*
|
|
223
|
+
* ## The metadata-only path, which is the whole point
|
|
224
|
+
*
|
|
225
|
+
* A columnar batch names its columns once per distinct key-set, in `shapes`, and
|
|
226
|
+
* carries the data in `values` as arrays that are *positional* — `values[i][3]`
|
|
227
|
+
* is whatever `shapes[shapeOf[i]][3]` is called. A positional array does not
|
|
228
|
+
* care what the key is called. So a pure rename rewrites `shapes` and hands back
|
|
229
|
+
* the **same `shapeOf` and the same `values` arrays, by reference**: a hundred
|
|
230
|
+
* thousand rows cost as many string comparisons as there are distinct key-sets,
|
|
231
|
+
* which for a real load is one or two.
|
|
232
|
+
*
|
|
233
|
+
* `dropUnnamed` breaks that and is allowed to. Removing a column removes a
|
|
234
|
+
* position, so every `values` row has to be rebuilt, and the result says so
|
|
235
|
+
* through {@link StageRenameResult.metadataOnly} rather than leaving the
|
|
236
|
+
* difference to be inferred from a stopwatch.
|
|
237
|
+
*
|
|
238
|
+
* ## Collisions
|
|
239
|
+
*
|
|
240
|
+
* A rename onto a name the shape already holds **throws**, naming both columns.
|
|
241
|
+
* The alternative is a shape with one name twice, which decodes to whichever
|
|
242
|
+
* value was written last — one of the author's two columns silently gone, with
|
|
243
|
+
* a green run. It is detected per shape rather than per row, so it fails on the
|
|
244
|
+
* first batch rather than at row ninety thousand.
|
|
245
|
+
*
|
|
246
|
+
* Under `dropUnnamed` there is nothing to collide with: a column the plan does
|
|
247
|
+
* not name is not in the output, so it cannot be occupying a name.
|
|
248
|
+
*
|
|
249
|
+
* ## A row-oriented batch
|
|
250
|
+
*
|
|
251
|
+
* Re-encoded first and then renamed by the one code path above, rather than
|
|
252
|
+
* given a second implementation that walks objects. Two implementations of "what
|
|
253
|
+
* does this rename mean" is how the fallback path ends up disagreeing with the
|
|
254
|
+
* fast one about a collision. The re-encode costs a pass and is reported as
|
|
255
|
+
* `metadataOnly: false`, which is the truth about the bytes.
|
|
256
|
+
*/
|
|
257
|
+
export declare function renameStagePayload(stored: unknown, plan: StageRenamePlan): StageRenameResult;
|
|
@@ -108,6 +108,7 @@ exports.encodeStageRows = encodeStageRows;
|
|
|
108
108
|
exports.isColumnarStageBatch = isColumnarStageBatch;
|
|
109
109
|
exports.classifyStagePayload = classifyStagePayload;
|
|
110
110
|
exports.decodeStageRows = decodeStageRows;
|
|
111
|
+
exports.renameStagePayload = renameStagePayload;
|
|
111
112
|
/** The tag every batch written by this build carries. */
|
|
112
113
|
exports.STAGE_ENCODING = 'columnar';
|
|
113
114
|
/**
|
|
@@ -293,6 +294,105 @@ function decodeStageRows(stored) {
|
|
|
293
294
|
}
|
|
294
295
|
}
|
|
295
296
|
}
|
|
297
|
+
/**
|
|
298
|
+
* Rename the columns of one staged batch.
|
|
299
|
+
*
|
|
300
|
+
* ## The metadata-only path, which is the whole point
|
|
301
|
+
*
|
|
302
|
+
* A columnar batch names its columns once per distinct key-set, in `shapes`, and
|
|
303
|
+
* carries the data in `values` as arrays that are *positional* — `values[i][3]`
|
|
304
|
+
* is whatever `shapes[shapeOf[i]][3]` is called. A positional array does not
|
|
305
|
+
* care what the key is called. So a pure rename rewrites `shapes` and hands back
|
|
306
|
+
* the **same `shapeOf` and the same `values` arrays, by reference**: a hundred
|
|
307
|
+
* thousand rows cost as many string comparisons as there are distinct key-sets,
|
|
308
|
+
* which for a real load is one or two.
|
|
309
|
+
*
|
|
310
|
+
* `dropUnnamed` breaks that and is allowed to. Removing a column removes a
|
|
311
|
+
* position, so every `values` row has to be rebuilt, and the result says so
|
|
312
|
+
* through {@link StageRenameResult.metadataOnly} rather than leaving the
|
|
313
|
+
* difference to be inferred from a stopwatch.
|
|
314
|
+
*
|
|
315
|
+
* ## Collisions
|
|
316
|
+
*
|
|
317
|
+
* A rename onto a name the shape already holds **throws**, naming both columns.
|
|
318
|
+
* The alternative is a shape with one name twice, which decodes to whichever
|
|
319
|
+
* value was written last — one of the author's two columns silently gone, with
|
|
320
|
+
* a green run. It is detected per shape rather than per row, so it fails on the
|
|
321
|
+
* first batch rather than at row ninety thousand.
|
|
322
|
+
*
|
|
323
|
+
* Under `dropUnnamed` there is nothing to collide with: a column the plan does
|
|
324
|
+
* not name is not in the output, so it cannot be occupying a name.
|
|
325
|
+
*
|
|
326
|
+
* ## A row-oriented batch
|
|
327
|
+
*
|
|
328
|
+
* Re-encoded first and then renamed by the one code path above, rather than
|
|
329
|
+
* given a second implementation that walks objects. Two implementations of "what
|
|
330
|
+
* does this rename mean" is how the fallback path ends up disagreeing with the
|
|
331
|
+
* fast one about a collision. The re-encode costs a pass and is reported as
|
|
332
|
+
* `metadataOnly: false`, which is the truth about the bytes.
|
|
333
|
+
*/
|
|
334
|
+
function renameStagePayload(stored, plan) {
|
|
335
|
+
const payload = classifyStagePayload(stored);
|
|
336
|
+
const wasColumnar = payload.encoding === 'columnar/v1';
|
|
337
|
+
const batch = wasColumnar ? payload.batch : encodeStageRows(payload.rows.filter(isRowRecord));
|
|
338
|
+
const matched = new Set();
|
|
339
|
+
const shapes = [];
|
|
340
|
+
const keeps = [];
|
|
341
|
+
let shapesRewritten = 0;
|
|
342
|
+
for (const shape of batch.shapes) {
|
|
343
|
+
const names = [];
|
|
344
|
+
const kept = [];
|
|
345
|
+
// Where each output name came from, so a collision can name both sides.
|
|
346
|
+
const placed = new Map();
|
|
347
|
+
for (const [at, name] of shape.entries()) {
|
|
348
|
+
const to = plan.columns.get(name);
|
|
349
|
+
if (to !== undefined)
|
|
350
|
+
matched.add(name);
|
|
351
|
+
if (to === undefined && plan.dropUnnamed)
|
|
352
|
+
continue;
|
|
353
|
+
const out = to ?? name;
|
|
354
|
+
const already = placed.get(out);
|
|
355
|
+
if (already !== undefined) {
|
|
356
|
+
throw new Error(`Renaming ${JSON.stringify(name)} to ${JSON.stringify(out)} would collide with ${JSON.stringify(already)}, which is already called that in the same rows. Two columns cannot share one name — one of them would silently win and the run would report success. Rename the other one too, or drop the columns this node does not name.`);
|
|
357
|
+
}
|
|
358
|
+
placed.set(out, name);
|
|
359
|
+
names.push(out);
|
|
360
|
+
kept.push(at);
|
|
361
|
+
}
|
|
362
|
+
shapes.push(names);
|
|
363
|
+
keeps.push(kept);
|
|
364
|
+
if (!sameKeys(names, shape))
|
|
365
|
+
shapesRewritten += 1;
|
|
366
|
+
}
|
|
367
|
+
// The pure-rename case: every position survived in place, so `values` is
|
|
368
|
+
// handed straight back. This is the line the node's whole argument rests on.
|
|
369
|
+
const metadataOnly = wasColumnar && !plan.dropUnnamed;
|
|
370
|
+
const values = metadataOnly
|
|
371
|
+
? batch.values
|
|
372
|
+
: batch.shapeOf.map((at, index) => pick(batch.values[index] ?? [], keeps[at] ?? []));
|
|
373
|
+
return {
|
|
374
|
+
payload: {
|
|
375
|
+
enc: exports.STAGE_ENCODING,
|
|
376
|
+
v: exports.STAGE_ENCODING_VERSION,
|
|
377
|
+
shapes,
|
|
378
|
+
shapeOf: batch.shapeOf,
|
|
379
|
+
values,
|
|
380
|
+
},
|
|
381
|
+
rows: batch.shapeOf.length,
|
|
382
|
+
metadataOnly,
|
|
383
|
+
shapesRewritten,
|
|
384
|
+
matched,
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
/** The named positions of one row, in order. */
|
|
388
|
+
function pick(cells, at) {
|
|
389
|
+
const out = new Array(at.length);
|
|
390
|
+
for (let index = 0; index < at.length; index += 1) {
|
|
391
|
+
const from = at[index];
|
|
392
|
+
out[index] = from === undefined ? null : cells[from];
|
|
393
|
+
}
|
|
394
|
+
return out;
|
|
395
|
+
}
|
|
296
396
|
function fromColumnar(batch) {
|
|
297
397
|
const rows = [];
|
|
298
398
|
for (const [index, at] of batch.shapeOf.entries()) {
|
package/dist/client.d.ts
CHANGED
|
@@ -153,9 +153,9 @@ export declare const catalogRoutes: {
|
|
|
153
153
|
readonly traces: () => string;
|
|
154
154
|
readonly trace: (id: string) => string;
|
|
155
155
|
};
|
|
156
|
-
export type { CatalogConnection, CatalogConnector, ConnectionCheck, CatalogTransform, CatalogTransformFunction, CatalogTransformInput, CatalogWorkflow, CatalogWorkflowCapabilities, CatalogWorkflowRelease, ConnectorKind, ConnectorRun, TransformLanguage, TransformResult, CallableWorkflowRef, WorkflowBranchLabel, WorkflowCallEnvelope, WorkflowCallMode, WorkflowCallNode, WorkflowCallOutput, WorkflowEdge, WorkflowExecutionMode, WorkflowFilterAll, WorkflowFilterAny, WorkflowFilterComparison, WorkflowFilterGroup, WorkflowFilterNode, WorkflowFilterOneOf, WorkflowFilterOperator, WorkflowFilterPredicate, WorkflowFilterPredicateKind, WorkflowFilterPresence, WorkflowFilterValue, WorkflowGraph, WorkflowEnvPredicate, WorkflowIfNode, WorkflowIfPredicate, WorkflowIssueCode, WorkflowNode, WorkflowNodeKind, WorkflowNodeOutcome, WorkflowPredicateKind, WorkflowRowCountPredicate, WorkflowRunOrderEntry, WorkflowSinkNode, WorkflowSkipReason, WorkflowSourceNode, WorkflowStageRef, WorkflowTransformNode, WorkflowValidationIssue, CatalogReusableNode, CatalogReusableNodeUse, ReusableNodeBody, ReusableNodeKind, ReusableNodeRef, ReusableSinkBody, ReusableSourceBody, SourceFormat, VersionPinCopy, } from './catalog.pipeline';
|
|
156
|
+
export type { CatalogConnection, CatalogConnector, ConnectionCheck, CatalogTransform, CatalogTransformFunction, CatalogTransformInput, CatalogRecordTransformFunction, CatalogRecordTransformInput, CatalogWorkflow, CatalogWorkflowCapabilities, CatalogWorkflowRelease, ConnectorKind, ConnectorRun, TransformLanguage, TransformMode, TransformResult, CallableWorkflowRef, WorkflowBranchLabel, WorkflowCallEnvelope, WorkflowCallMode, WorkflowCallNode, WorkflowCallOutput, WorkflowEdge, WorkflowExecutionMode, WorkflowFilterAll, WorkflowFilterAny, WorkflowFilterComparison, WorkflowFilterGroup, WorkflowFilterNode, WorkflowFilterOneOf, WorkflowFilterOperator, WorkflowFilterPredicate, WorkflowFilterPredicateKind, WorkflowFilterPresence, WorkflowFilterValue, WorkflowGraph, WorkflowEnvPredicate, WorkflowIfNode, WorkflowIfPredicate, WorkflowIssueCode, WorkflowNode, WorkflowNodeKind, WorkflowNodeOutcome, WorkflowPredicateKind, WorkflowRenameNode, WorkflowRenameUnnamed, WorkflowRowCountPredicate, WorkflowRunOrderEntry, WorkflowSinkNode, WorkflowSkipReason, WorkflowSourceNode, WorkflowStageRef, WorkflowTransformNode, WorkflowValidationIssue, CatalogReusableNode, CatalogReusableNodeUse, ReusableNodeBody, ReusableNodeKind, ReusableNodeRef, ReusableSinkBody, ReusableSourceBody, SourceFormat, VersionPinCopy, } from './catalog.pipeline';
|
|
157
157
|
export { type TransformShape, transformDeclaresModule, transformShape, } from './transform-shape';
|
|
158
|
-
export { CONNECTOR_KINDS, isConnectorKind, isSourceFormat, isTransformLanguage, SOURCE_FORMATS, isWorkflowEdge, isWorkflowNode, liveWorkflowVersion, REDACTED_SECRET, TRANSFORM_LANGUAGES, readWorkflowCallOutput, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, isWorkflowCallMode, unreachableCallMode, workflowCallMode, applyReusableNode, reusableNodeBodyOf, isReusableNodeBody, isReusableNodeKind, REUSABLE_NODE_KINDS, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, unreachableReusableNodeKind, describeLiveVersion, describeVersionPin, } from './catalog.pipeline';
|
|
158
|
+
export { CONNECTOR_KINDS, isConnectorKind, isSourceFormat, isTransformLanguage, isTransformMode, TRANSFORM_MODES, transformMode, recordModeRefusal, SOURCE_FORMATS, isWorkflowEdge, isWorkflowNode, liveWorkflowVersion, REDACTED_SECRET, TRANSFORM_LANGUAGES, readWorkflowCallOutput, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, isWorkflowCallMode, unreachableCallMode, workflowCallMode, applyReusableNode, reusableNodeBodyOf, isReusableNodeBody, isReusableNodeKind, REUSABLE_NODE_KINDS, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, unreachableReusableNodeKind, describeLiveVersion, describeVersionPin, } from './catalog.pipeline';
|
|
159
159
|
/**
|
|
160
160
|
* The workflow validator, shipped to the browser deliberately.
|
|
161
161
|
*
|
|
@@ -183,7 +183,7 @@ export { CONNECTOR_KINDS, isConnectorKind, isSourceFormat, isTransformLanguage,
|
|
|
183
183
|
* ends up drawn with its boxes overlapping — which is exactly what happened.
|
|
184
184
|
*/
|
|
185
185
|
export { WORKFLOW_COLUMN_GAP, WORKFLOW_NODE_HEIGHT, WORKFLOW_NODE_WIDTH, WORKFLOW_ROW_GAP, workflowColumnX, workflowRowY, } from './catalog.pipeline';
|
|
186
|
-
export { validateWorkflow, WORKFLOW_EXECUTION_MODES, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_BRANCH_LABELS, WORKFLOW_PREDICATE_KINDS, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_SKIP_REASONS, isWorkflowBranchLabel, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowPredicateKind, isWorkflowSkipReason, workflowFilterMatches, workflowNarrowedTypes, workflowNodeRuns, unreachableFilterOperator, unreachableFilterPredicateKind, unreachableNodeKind, unreachablePredicateKind, WORKFLOW_STATUSES, workflowGraphHash, workflowRunOrder, isWorkflowExecutionMode, isWorkflowNodeKind, isWorkflowStatus, callableWorkflowBlock, } from './catalog.pipeline';
|
|
186
|
+
export { validateWorkflow, WORKFLOW_EXECUTION_MODES, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_BRANCH_LABELS, WORKFLOW_PREDICATE_KINDS, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_RENAME_MAX_COLUMNS, WORKFLOW_RENAME_UNNAMED, WORKFLOW_SKIP_REASONS, isWorkflowBranchLabel, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowPredicateKind, isWorkflowRenameColumns, isWorkflowRenameUnnamed, isWorkflowSkipReason, workflowFilterColumns, workflowFilterMatches, renameColumnRefusals, workflowRenameUnnamed, workflowKnownColumns, workflowNarrowedTypes, workflowNodeRuns, unreachableFilterOperator, unreachableFilterPredicateKind, unreachableNodeKind, unreachablePredicateKind, unreachableRenameUnnamed, WORKFLOW_STATUSES, workflowGraphHash, workflowRunOrder, isWorkflowExecutionMode, isWorkflowNodeKind, isWorkflowStatus, callableWorkflowBlock, } from './catalog.pipeline';
|
|
187
187
|
export type { CallableWorkflowBlock, CallableWorkflowDisagreement, WorkflowStatus, } from './catalog.pipeline';
|
|
188
188
|
export type { CatalogTrace, CatalogTraceList, CatalogTraceOutcome, CatalogTraceSpan, TraceQuery, } from './catalog.workspace';
|
|
189
189
|
export type { CatalogLoadExpectations, DeleteReconciliation, LoadExpectation, RowCountBound, StoredLoadExpectation, } from './catalog.pipeline';
|
package/dist/client.js
CHANGED
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* types are.
|
|
12
12
|
*/
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
-
exports.
|
|
15
|
-
exports.DELETE_RECONCILIATION_STRATEGIES = exports.callableWorkflowBlock = exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableFilterPredicateKind = exports.unreachableFilterOperator = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowFilterMatches = exports.isWorkflowSkipReason = exports.isWorkflowPredicateKind = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowBranchLabel = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.workflowRowY = void 0;
|
|
14
|
+
exports.WORKFLOW_COLUMN_GAP = exports.describeVersionPin = exports.describeLiveVersion = exports.unreachableReusableNodeKind = exports.nodeKindIsReusable = exports.NODE_KIND_IS_REUSABLE = exports.REUSABLE_NODE_KINDS = exports.isReusableNodeKind = exports.isReusableNodeBody = exports.reusableNodeBodyOf = exports.applyReusableNode = exports.workflowCallMode = exports.unreachableCallMode = exports.isWorkflowCallMode = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_CALL_CONTRACT = exports.readWorkflowCallOutput = exports.TRANSFORM_LANGUAGES = exports.REDACTED_SECRET = exports.liveWorkflowVersion = exports.isWorkflowNode = exports.isWorkflowEdge = exports.SOURCE_FORMATS = exports.recordModeRefusal = exports.transformMode = exports.TRANSFORM_MODES = exports.isTransformMode = exports.isTransformLanguage = exports.isSourceFormat = exports.isConnectorKind = exports.CONNECTOR_KINDS = exports.transformShape = exports.transformDeclaresModule = exports.catalogRoutes = exports.UnsafeIdentifierError = exports.physicalColumn = exports.outputAlias = exports.isSafeIdentifier = exports.VALUELESS_FILTER_OPERATORS = exports.resolveObjectFilters = exports.parseObjectFilter = exports.offeredFilterOperators = exports.isCatalogFilterOperator = exports.filterOperatorsFor = exports.filterOperatorTakesValue = exports.encodeObjectFilter = exports.coerceFilterValue = exports.CATALOG_FILTER_OPERATORS = exports.CATALOG_FILTER_LIMIT = exports.CATALOG_REVISION_LIMIT = void 0;
|
|
15
|
+
exports.DELETE_RECONCILIATION_STRATEGIES = exports.callableWorkflowBlock = exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.unreachableRenameUnnamed = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableFilterPredicateKind = exports.unreachableFilterOperator = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowKnownColumns = exports.workflowRenameUnnamed = exports.renameColumnRefusals = exports.workflowFilterMatches = exports.workflowFilterColumns = exports.isWorkflowSkipReason = exports.isWorkflowRenameUnnamed = exports.isWorkflowRenameColumns = exports.isWorkflowPredicateKind = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowBranchLabel = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_RENAME_UNNAMED = exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.workflowRowY = exports.workflowColumnX = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_HEIGHT = void 0;
|
|
16
16
|
exports.isDeleteReconciliationStrategy = isDeleteReconciliationStrategy;
|
|
17
17
|
exports.pipelineExpectationRoutes = pipelineExpectationRoutes;
|
|
18
18
|
// A value, not a type: a screen saying how far back the history goes should read
|
|
@@ -144,6 +144,17 @@ Object.defineProperty(exports, "CONNECTOR_KINDS", { enumerable: true, get: funct
|
|
|
144
144
|
Object.defineProperty(exports, "isConnectorKind", { enumerable: true, get: function () { return catalog_pipeline_1.isConnectorKind; } });
|
|
145
145
|
Object.defineProperty(exports, "isSourceFormat", { enumerable: true, get: function () { return catalog_pipeline_1.isSourceFormat; } });
|
|
146
146
|
Object.defineProperty(exports, "isTransformLanguage", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformLanguage; } });
|
|
147
|
+
// The editor offers exactly the modes the runner can call, and applies the
|
|
148
|
+
// default through the same function the store and both runners do. A screen
|
|
149
|
+
// carrying its own `?? 'batch'` is the second copy that decides a stored
|
|
150
|
+
// transform means something else.
|
|
151
|
+
Object.defineProperty(exports, "isTransformMode", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformMode; } });
|
|
152
|
+
Object.defineProperty(exports, "TRANSFORM_MODES", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_MODES; } });
|
|
153
|
+
Object.defineProperty(exports, "transformMode", { enumerable: true, get: function () { return catalog_pipeline_1.transformMode; } });
|
|
154
|
+
// Whether the two impossible combinations are impossible, asked while the
|
|
155
|
+
// author is still looking at the code rather than at three in the morning when
|
|
156
|
+
// a schedule fires.
|
|
157
|
+
Object.defineProperty(exports, "recordModeRefusal", { enumerable: true, get: function () { return catalog_pipeline_1.recordModeRefusal; } });
|
|
147
158
|
Object.defineProperty(exports, "SOURCE_FORMATS", { enumerable: true, get: function () { return catalog_pipeline_1.SOURCE_FORMATS; } });
|
|
148
159
|
// The canvas narrows nodes and edges it reads back from HTTP. Without these
|
|
149
160
|
// it either imports them from the package root — dragging NestJS and MikroORM
|
|
@@ -263,6 +274,11 @@ Object.defineProperty(exports, "WORKFLOW_FILTER_MAX_DEPTH", { enumerable: true,
|
|
|
263
274
|
Object.defineProperty(exports, "WORKFLOW_FILTER_MAX_VALUES", { enumerable: true, get: function () { return catalog_pipeline_3.WORKFLOW_FILTER_MAX_VALUES; } });
|
|
264
275
|
Object.defineProperty(exports, "WORKFLOW_FILTER_OPERATORS", { enumerable: true, get: function () { return catalog_pipeline_3.WORKFLOW_FILTER_OPERATORS; } });
|
|
265
276
|
Object.defineProperty(exports, "WORKFLOW_FILTER_PREDICATE_KINDS", { enumerable: true, get: function () { return catalog_pipeline_3.WORKFLOW_FILTER_PREDICATE_KINDS; } });
|
|
277
|
+
// The rename vocabulary, same argument: the inspector has to refuse a target
|
|
278
|
+
// name the server would refuse, and the two words for what happens to an
|
|
279
|
+
// unnamed column have to be exactly the two the runner branches on.
|
|
280
|
+
Object.defineProperty(exports, "WORKFLOW_RENAME_MAX_COLUMNS", { enumerable: true, get: function () { return catalog_pipeline_3.WORKFLOW_RENAME_MAX_COLUMNS; } });
|
|
281
|
+
Object.defineProperty(exports, "WORKFLOW_RENAME_UNNAMED", { enumerable: true, get: function () { return catalog_pipeline_3.WORKFLOW_RENAME_UNNAMED; } });
|
|
266
282
|
Object.defineProperty(exports, "WORKFLOW_SKIP_REASONS", { enumerable: true, get: function () { return catalog_pipeline_3.WORKFLOW_SKIP_REASONS; } });
|
|
267
283
|
Object.defineProperty(exports, "isWorkflowBranchLabel", { enumerable: true, get: function () { return catalog_pipeline_3.isWorkflowBranchLabel; } });
|
|
268
284
|
Object.defineProperty(exports, "isWorkflowFilterOperator", { enumerable: true, get: function () { return catalog_pipeline_3.isWorkflowFilterOperator; } });
|
|
@@ -271,10 +287,22 @@ Object.defineProperty(exports, "isWorkflowFilterPredicateKind", { enumerable: tr
|
|
|
271
287
|
Object.defineProperty(exports, "isWorkflowFilterValue", { enumerable: true, get: function () { return catalog_pipeline_3.isWorkflowFilterValue; } });
|
|
272
288
|
Object.defineProperty(exports, "isWorkflowIfPredicate", { enumerable: true, get: function () { return catalog_pipeline_3.isWorkflowIfPredicate; } });
|
|
273
289
|
Object.defineProperty(exports, "isWorkflowPredicateKind", { enumerable: true, get: function () { return catalog_pipeline_3.isWorkflowPredicateKind; } });
|
|
290
|
+
Object.defineProperty(exports, "isWorkflowRenameColumns", { enumerable: true, get: function () { return catalog_pipeline_3.isWorkflowRenameColumns; } });
|
|
291
|
+
Object.defineProperty(exports, "isWorkflowRenameUnnamed", { enumerable: true, get: function () { return catalog_pipeline_3.isWorkflowRenameUnnamed; } });
|
|
274
292
|
Object.defineProperty(exports, "isWorkflowSkipReason", { enumerable: true, get: function () { return catalog_pipeline_3.isWorkflowSkipReason; } });
|
|
275
293
|
// The row test itself, so the inspector can describe — and a host can preview
|
|
276
294
|
// — exactly what a load will keep, from the function that decides it.
|
|
295
|
+
Object.defineProperty(exports, "workflowFilterColumns", { enumerable: true, get: function () { return catalog_pipeline_3.workflowFilterColumns; } });
|
|
277
296
|
Object.defineProperty(exports, "workflowFilterMatches", { enumerable: true, get: function () { return catalog_pipeline_3.workflowFilterMatches; } });
|
|
297
|
+
// The refusals a rename map earns, from the function the validator and the
|
|
298
|
+
// HTTP boundary both call. A form with its own copy of the identifier pattern
|
|
299
|
+
// is a form that eventually accepts a target the server refuses, after Save.
|
|
300
|
+
Object.defineProperty(exports, "renameColumnRefusals", { enumerable: true, get: function () { return catalog_pipeline_3.renameColumnRefusals; } });
|
|
301
|
+
Object.defineProperty(exports, "workflowRenameUnnamed", { enumerable: true, get: function () { return catalog_pipeline_3.workflowRenameUnnamed; } });
|
|
302
|
+
// What the graph can prove about the columns reaching a node — the one thing
|
|
303
|
+
// a declarative rename buys that a transform cannot. The inspector says it out
|
|
304
|
+
// loud; see `workflowKnownColumns` for how far it reaches.
|
|
305
|
+
Object.defineProperty(exports, "workflowKnownColumns", { enumerable: true, get: function () { return catalog_pipeline_3.workflowKnownColumns; } });
|
|
278
306
|
// Which published types a filter stands in front of. The console has to offer
|
|
279
307
|
// the same acknowledgements the validator requires, and a canvas computing its
|
|
280
308
|
// own answer would offer a set the server then refuses.
|
|
@@ -286,6 +314,7 @@ Object.defineProperty(exports, "unreachableFilterOperator", { enumerable: true,
|
|
|
286
314
|
Object.defineProperty(exports, "unreachableFilterPredicateKind", { enumerable: true, get: function () { return catalog_pipeline_3.unreachableFilterPredicateKind; } });
|
|
287
315
|
Object.defineProperty(exports, "unreachableNodeKind", { enumerable: true, get: function () { return catalog_pipeline_3.unreachableNodeKind; } });
|
|
288
316
|
Object.defineProperty(exports, "unreachablePredicateKind", { enumerable: true, get: function () { return catalog_pipeline_3.unreachablePredicateKind; } });
|
|
317
|
+
Object.defineProperty(exports, "unreachableRenameUnnamed", { enumerable: true, get: function () { return catalog_pipeline_3.unreachableRenameUnnamed; } });
|
|
289
318
|
// The draft/ready pair, for the same reason as the list above: a canvas that
|
|
290
319
|
// cannot see it restates it, and the copy is what drifts. Without this the
|
|
291
320
|
// editor could not tell a graph it is allowed to store from one the server
|
package/dist/index.d.ts
CHANGED
|
@@ -8,8 +8,8 @@ export { type CatalogOverlayStore, FileCatalogOverlayStore, InMemoryCatalogOverl
|
|
|
8
8
|
export { CATALOG_OVERLAY_STORE } from './catalog.overlay-store.token';
|
|
9
9
|
export { MikroOrmCatalogRegistry } from './catalog.registry';
|
|
10
10
|
export { CatalogRegistry } from './catalog.registry.base';
|
|
11
|
-
export { CATALOG_PIPELINE_STORE, CODE_CONTEXT_CONTRACT, type CatalogCodeContext, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogLoadExpectations, type CatalogLoadExpectationStore, type CatalogPipelineStore, type CatalogStageStore, type CatalogTransform, type CatalogTransformFunction, type CatalogTransformInput, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowRelease, type CatalogWorkflowReleaseStore, type CatalogWorkflowStore, type CallableWorkflowBlock, type CallableWorkflowDisagreement, type CallableWorkflowRef, callableWorkflowBlock, type ConnectorKind, type ConnectorRun, type DeleteReconciliation, isConnectorKind, isPipelineStore, isSourceFormat, isTransformLanguage, type LoadExpectation, type RowCountBound, SOURCE_FORMATS, type SourceFormat, unreachableSourceFormat, type StoredLoadExpectation, readWorkflowCallOutput, REDACTED_SECRET, supportsLoadExpectations, supportsReusableNodes, supportsTransformPins, supportsTransformRevisions, applyReusableNode, type CatalogReusableNode, type CatalogReusableNodeStore, type CatalogReusableNodeUse, describeLiveVersion, describeVersionPin, isReusableNodeBody, isReusableNodeKind, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, REUSABLE_NODE_KINDS, type ReusableNodeBody, type ReusableNodeKind, type ReusableNodeRef, type ReusableSinkBody, type ReusableSourceBody, reusableNodeBodyOf, unreachableReusableNodeKind, type VersionPinCopy, isWorkflowBranchLabel, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowNode, isWorkflowCallMode, isWorkflowNodeKind, isWorkflowPredicateKind, isWorkflowSkipReason, isWorkflowStatus, liveWorkflowVersion, supportsWorkflowReleases, supportsWorkflows, supportsWorkflowStages, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, type TransformLanguage, type TransformResult, type TransformRunner, unreachableFilterOperator, unreachableFilterPredicateKind, unreachableCallMode, unreachableNodeKind, unreachablePredicateKind, validateWorkflow, WORKFLOW_BRANCH_LABELS, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, WORKFLOW_COLUMN_GAP, WORKFLOW_EXECUTION_MODES, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_HEIGHT, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_NODE_WIDTH, WORKFLOW_PREDICATE_KINDS, WORKFLOW_ROW_GAP, WORKFLOW_SKIP_REASONS, WORKFLOW_STATUSES, type WorkflowBranchLabel, workflowColumnX, workflowRowY, type WorkflowCallEnvelope, type WorkflowCallMode, workflowCallMode, type WorkflowCallNode, type WorkflowCallOutput, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowFilterAll, type WorkflowFilterAny, type WorkflowFilterComparison, type WorkflowFilterGroup, workflowFilterMatches, type WorkflowFilterNode, type WorkflowFilterOneOf, type WorkflowFilterOperator, type WorkflowFilterPredicate, type WorkflowFilterPredicateKind, type WorkflowFilterPresence, type WorkflowFilterValue, type WorkflowGraph, workflowGraphHash, type WorkflowEnvPredicate, type WorkflowIfNode, type WorkflowIfPredicate, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, workflowNarrowedTypes, type WorkflowNodeOutcome, workflowNodeRuns, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowRunOrderEntry, type WorkflowPredicateKind, type WorkflowRowCountPredicate, type WorkflowSinkNode, type WorkflowSkipReason, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowStatus, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
|
|
12
|
-
export { type ColumnarStageBatch, STAGE_ENCODING, STAGE_ENCODING_VERSION, type StagePayload, classifyStagePayload, decodeStageRows, encodeStageRows, isColumnarStageBatch, } from './catalog.stage-encoding';
|
|
11
|
+
export { CATALOG_PIPELINE_STORE, CODE_CONTEXT_CONTRACT, type CatalogCodeContext, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogLoadExpectations, type CatalogLoadExpectationStore, type CatalogPipelineStore, type CatalogStageStore, type CatalogRecordTransformFunction, type CatalogRecordTransformInput, type CatalogTransform, type CatalogTransformFunction, type CatalogTransformInput, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowRelease, type CatalogWorkflowReleaseStore, type CatalogWorkflowStore, type CallableWorkflowBlock, type CallableWorkflowDisagreement, type CallableWorkflowRef, callableWorkflowBlock, type ConnectorKind, type ConnectorRun, type DeleteReconciliation, isConnectorKind, isPipelineStore, isSourceFormat, isTransformLanguage, isTransformMode, type LoadExpectation, type RowCountBound, SOURCE_FORMATS, type SourceFormat, unreachableSourceFormat, type StoredLoadExpectation, readWorkflowCallOutput, REDACTED_SECRET, supportsLoadExpectations, supportsReusableNodes, supportsTransformPins, supportsTransformRevisions, supportsTransformStreaming, applyReusableNode, type CatalogReusableNode, type CatalogReusableNodeStore, type CatalogReusableNodeUse, describeLiveVersion, describeVersionPin, isReusableNodeBody, isReusableNodeKind, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, REUSABLE_NODE_KINDS, type ReusableNodeBody, type ReusableNodeKind, type ReusableNodeRef, type ReusableSinkBody, type ReusableSourceBody, reusableNodeBodyOf, unreachableReusableNodeKind, type VersionPinCopy, isWorkflowBranchLabel, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowNode, isWorkflowCallMode, isWorkflowNodeKind, isWorkflowPredicateKind, isWorkflowRenameColumns, isWorkflowRenameUnnamed, isWorkflowSkipReason, isWorkflowStatus, liveWorkflowVersion, supportsWorkflowReleases, supportsWorkflows, supportsWorkflowStages, supportsStagePayloads, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, TRANSFORM_MODES, recordModeRefusal, transformMode, type TransformLanguage, type TransformMode, type TransformResult, type TransformRunner, type TransformStream, type TransformStreamSummary, unreachableFilterOperator, unreachableTransformMode, unreachableFilterPredicateKind, unreachableCallMode, unreachableNodeKind, unreachablePredicateKind, unreachableRenameUnnamed, validateWorkflow, WORKFLOW_BRANCH_LABELS, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, WORKFLOW_COLUMN_GAP, WORKFLOW_EXECUTION_MODES, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_HEIGHT, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_NODE_WIDTH, WORKFLOW_PREDICATE_KINDS, WORKFLOW_RENAME_MAX_COLUMNS, WORKFLOW_RENAME_UNNAMED, WORKFLOW_ROW_GAP, WORKFLOW_SKIP_REASONS, WORKFLOW_STATUSES, type WorkflowBranchLabel, workflowColumnX, workflowRowY, type WorkflowCallEnvelope, type WorkflowCallMode, workflowCallMode, type WorkflowCallNode, type WorkflowCallOutput, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowFilterAll, type WorkflowFilterAny, type WorkflowFilterComparison, type WorkflowFilterGroup, workflowFilterColumns, workflowFilterMatches, type WorkflowFilterNode, type WorkflowFilterOneOf, type WorkflowFilterOperator, type WorkflowFilterPredicate, type WorkflowFilterPredicateKind, type WorkflowFilterPresence, type WorkflowFilterValue, type WorkflowGraph, workflowGraphHash, type WorkflowEnvPredicate, type WorkflowIfNode, type WorkflowIfPredicate, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, workflowKnownColumns, workflowNarrowedTypes, type WorkflowNodeOutcome, workflowNodeRuns, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowRunOrderEntry, type WorkflowPredicateKind, renameColumnRefusals, type WorkflowRenameNode, type WorkflowRenameUnnamed, workflowRenameUnnamed, type WorkflowRowCountPredicate, type WorkflowSinkNode, type WorkflowSkipReason, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowStatus, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
|
|
12
|
+
export { type ColumnarStageBatch, STAGE_ENCODING, STAGE_ENCODING_VERSION, type StagePayload, classifyStagePayload, decodeStageRows, encodeStageRows, isColumnarStageBatch, renameStagePayload, type StageRenamePlan, type StageRenameResult, } from './catalog.stage-encoding';
|
|
13
13
|
export * from './catalog.environment';
|
|
14
14
|
export { QueryCache } from './catalog.query-cache';
|
|
15
15
|
export { type CsvRow, csvCell, csvLines, guardFormula, toCsv } from './catalog.csv';
|
package/dist/index.js
CHANGED
|
@@ -14,10 +14,10 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.
|
|
18
|
-
exports.
|
|
19
|
-
exports.
|
|
20
|
-
exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.UnsafeIdentifierError = exports.supportsCarryForward = exports.physicalColumn = exports.outputAlias = exports.isWriteStore = exports.isSafeIdentifier = exports.isReservedColumn = exports.isCatalogStoreCapabilities = exports.findColumnCollisions = exports.supportsObjectFilters = exports.CatalogColumnCollisionError = void 0;
|
|
17
|
+
exports.isWorkflowBranchLabel = exports.unreachableReusableNodeKind = exports.reusableNodeBodyOf = exports.REUSABLE_NODE_KINDS = exports.nodeKindIsReusable = exports.NODE_KIND_IS_REUSABLE = exports.isReusableNodeKind = exports.isReusableNodeBody = exports.describeVersionPin = exports.describeLiveVersion = exports.applyReusableNode = exports.supportsTransformStreaming = exports.supportsTransformRevisions = exports.supportsTransformPins = exports.supportsReusableNodes = exports.supportsLoadExpectations = exports.REDACTED_SECRET = exports.readWorkflowCallOutput = exports.unreachableSourceFormat = exports.SOURCE_FORMATS = exports.isTransformMode = exports.isTransformLanguage = exports.isSourceFormat = exports.isPipelineStore = exports.isConnectorKind = exports.callableWorkflowBlock = exports.CONNECTOR_KINDS = exports.CODE_CONTEXT_CONTRACT = exports.CATALOG_PIPELINE_STORE = exports.CatalogRegistry = exports.MikroOrmCatalogRegistry = exports.CATALOG_OVERLAY_STORE = exports.InMemoryCatalogOverlayStore = exports.FileCatalogOverlayStore = exports.CATALOG_OPTIONS = exports.isStreamingQueryStore = exports.isQueryStore = exports.assertReadOnlyShape = exports.CatalogModule = exports.emitCatalog = exports.curationActor = exports.channelNameFor = exports.catalogEventPhase = exports.UNATTRIBUTED_PRINCIPAL_ID = exports.CATALOG_LIB = exports.CATALOG_EVENTS = exports.CATALOG_EVENT_PHASE_FALLBACK = exports.CATALOG_EVENT_PHASE = exports.CatalogType = exports.CatalogProperty = void 0;
|
|
18
|
+
exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_BRANCH_LABELS = exports.validateWorkflow = exports.unreachableRenameUnnamed = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableCallMode = exports.unreachableFilterPredicateKind = exports.unreachableTransformMode = exports.unreachableFilterOperator = exports.transformMode = exports.recordModeRefusal = exports.TRANSFORM_MODES = exports.TRANSFORM_LANGUAGES = exports.TRANSFORM_RUNNER = exports.supportsStagePayloads = exports.supportsWorkflowStages = exports.supportsWorkflows = exports.supportsWorkflowReleases = exports.liveWorkflowVersion = exports.isWorkflowStatus = exports.isWorkflowSkipReason = exports.isWorkflowRenameUnnamed = exports.isWorkflowRenameColumns = exports.isWorkflowPredicateKind = exports.isWorkflowNodeKind = exports.isWorkflowCallMode = exports.isWorkflowNode = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowExecutionMode = exports.isWorkflowEdge = void 0;
|
|
19
|
+
exports.traceOutcomeFilter = exports.supportsSavedQueryRevisions = exports.isWorkspaceStore = exports.isTraceStore = exports.isCatalogTraceOutcome = exports.embeddedVisualization = exports.CATALOG_WORKSPACE_STORE = exports.CATALOG_TRACE_STORE = exports.CATALOG_TRACE_OUTCOMES = exports.CATALOG_REVISION_LIMIT = exports.visibleToPrincipal = exports.searchCatalog = exports.maySearch = exports.emptySearch = exports.bestMatch = exports.MAX_SEARCH_LIMIT = exports.DEFAULT_SEARCH_LIMIT = exports.CatalogService = exports.transformShapeHint = exports.transformShape = exports.transformDeclaresModule = exports.SubprocessTransformRunner = exports.toCsv = exports.guardFormula = exports.csvLines = exports.csvCell = exports.QueryCache = exports.renameStagePayload = exports.isColumnarStageBatch = exports.encodeStageRows = exports.decodeStageRows = exports.classifyStagePayload = exports.STAGE_ENCODING_VERSION = exports.STAGE_ENCODING = exports.workflowRenameUnnamed = exports.renameColumnRefusals = exports.workflowRunOrder = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowKnownColumns = exports.workflowGraphHash = exports.workflowFilterMatches = exports.workflowFilterColumns = exports.workflowCallMode = exports.workflowRowY = exports.workflowColumnX = exports.WORKFLOW_STATUSES = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_RENAME_UNNAMED = void 0;
|
|
20
|
+
exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.UnsafeIdentifierError = exports.supportsCarryForward = exports.physicalColumn = exports.outputAlias = exports.isWriteStore = exports.isSafeIdentifier = exports.isReservedColumn = exports.isCatalogStoreCapabilities = exports.findColumnCollisions = exports.supportsObjectFilters = exports.CatalogColumnCollisionError = exports.CATALOG_STORE = exports.CATALOG_SNAPSHOT_MODES = exports.CATALOG_RESERVED_COLUMNS = exports.assertSafeIdentifier = exports.assertNoColumnCollisions = exports.StaticKeyPrincipalResolver = exports.readableObjectPage = exports.mayWrite = exports.mayRead = exports.maySeeClassification = exports.PRINCIPAL_ACTOR_SEPARATOR = exports.parsePrincipalId = exports.hasScope = exports.expandScopes = exports.delegatePrincipal = exports.composePrincipalId = exports.CATALOG_PRINCIPAL_RESOLVER = void 0;
|
|
21
21
|
var catalog_decorators_1 = require("./catalog.decorators");
|
|
22
22
|
Object.defineProperty(exports, "CatalogProperty", { enumerable: true, get: function () { return catalog_decorators_1.CatalogProperty; } });
|
|
23
23
|
Object.defineProperty(exports, "CatalogType", { enumerable: true, get: function () { return catalog_decorators_1.CatalogType; } });
|
|
@@ -65,6 +65,7 @@ Object.defineProperty(exports, "isConnectorKind", { enumerable: true, get: funct
|
|
|
65
65
|
Object.defineProperty(exports, "isPipelineStore", { enumerable: true, get: function () { return catalog_pipeline_1.isPipelineStore; } });
|
|
66
66
|
Object.defineProperty(exports, "isSourceFormat", { enumerable: true, get: function () { return catalog_pipeline_1.isSourceFormat; } });
|
|
67
67
|
Object.defineProperty(exports, "isTransformLanguage", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformLanguage; } });
|
|
68
|
+
Object.defineProperty(exports, "isTransformMode", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformMode; } });
|
|
68
69
|
Object.defineProperty(exports, "SOURCE_FORMATS", { enumerable: true, get: function () { return catalog_pipeline_1.SOURCE_FORMATS; } });
|
|
69
70
|
Object.defineProperty(exports, "unreachableSourceFormat", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableSourceFormat; } });
|
|
70
71
|
Object.defineProperty(exports, "readWorkflowCallOutput", { enumerable: true, get: function () { return catalog_pipeline_1.readWorkflowCallOutput; } });
|
|
@@ -73,6 +74,7 @@ Object.defineProperty(exports, "supportsLoadExpectations", { enumerable: true, g
|
|
|
73
74
|
Object.defineProperty(exports, "supportsReusableNodes", { enumerable: true, get: function () { return catalog_pipeline_1.supportsReusableNodes; } });
|
|
74
75
|
Object.defineProperty(exports, "supportsTransformPins", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformPins; } });
|
|
75
76
|
Object.defineProperty(exports, "supportsTransformRevisions", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformRevisions; } });
|
|
77
|
+
Object.defineProperty(exports, "supportsTransformStreaming", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformStreaming; } });
|
|
76
78
|
Object.defineProperty(exports, "applyReusableNode", { enumerable: true, get: function () { return catalog_pipeline_1.applyReusableNode; } });
|
|
77
79
|
Object.defineProperty(exports, "describeLiveVersion", { enumerable: true, get: function () { return catalog_pipeline_1.describeLiveVersion; } });
|
|
78
80
|
Object.defineProperty(exports, "describeVersionPin", { enumerable: true, get: function () { return catalog_pipeline_1.describeVersionPin; } });
|
|
@@ -95,19 +97,27 @@ Object.defineProperty(exports, "isWorkflowNode", { enumerable: true, get: functi
|
|
|
95
97
|
Object.defineProperty(exports, "isWorkflowCallMode", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowCallMode; } });
|
|
96
98
|
Object.defineProperty(exports, "isWorkflowNodeKind", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowNodeKind; } });
|
|
97
99
|
Object.defineProperty(exports, "isWorkflowPredicateKind", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowPredicateKind; } });
|
|
100
|
+
Object.defineProperty(exports, "isWorkflowRenameColumns", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowRenameColumns; } });
|
|
101
|
+
Object.defineProperty(exports, "isWorkflowRenameUnnamed", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowRenameUnnamed; } });
|
|
98
102
|
Object.defineProperty(exports, "isWorkflowSkipReason", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowSkipReason; } });
|
|
99
103
|
Object.defineProperty(exports, "isWorkflowStatus", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowStatus; } });
|
|
100
104
|
Object.defineProperty(exports, "liveWorkflowVersion", { enumerable: true, get: function () { return catalog_pipeline_1.liveWorkflowVersion; } });
|
|
101
105
|
Object.defineProperty(exports, "supportsWorkflowReleases", { enumerable: true, get: function () { return catalog_pipeline_1.supportsWorkflowReleases; } });
|
|
102
106
|
Object.defineProperty(exports, "supportsWorkflows", { enumerable: true, get: function () { return catalog_pipeline_1.supportsWorkflows; } });
|
|
103
107
|
Object.defineProperty(exports, "supportsWorkflowStages", { enumerable: true, get: function () { return catalog_pipeline_1.supportsWorkflowStages; } });
|
|
108
|
+
Object.defineProperty(exports, "supportsStagePayloads", { enumerable: true, get: function () { return catalog_pipeline_1.supportsStagePayloads; } });
|
|
104
109
|
Object.defineProperty(exports, "TRANSFORM_RUNNER", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_RUNNER; } });
|
|
105
110
|
Object.defineProperty(exports, "TRANSFORM_LANGUAGES", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_LANGUAGES; } });
|
|
111
|
+
Object.defineProperty(exports, "TRANSFORM_MODES", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_MODES; } });
|
|
112
|
+
Object.defineProperty(exports, "recordModeRefusal", { enumerable: true, get: function () { return catalog_pipeline_1.recordModeRefusal; } });
|
|
113
|
+
Object.defineProperty(exports, "transformMode", { enumerable: true, get: function () { return catalog_pipeline_1.transformMode; } });
|
|
106
114
|
Object.defineProperty(exports, "unreachableFilterOperator", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableFilterOperator; } });
|
|
115
|
+
Object.defineProperty(exports, "unreachableTransformMode", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableTransformMode; } });
|
|
107
116
|
Object.defineProperty(exports, "unreachableFilterPredicateKind", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableFilterPredicateKind; } });
|
|
108
117
|
Object.defineProperty(exports, "unreachableCallMode", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableCallMode; } });
|
|
109
118
|
Object.defineProperty(exports, "unreachableNodeKind", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableNodeKind; } });
|
|
110
119
|
Object.defineProperty(exports, "unreachablePredicateKind", { enumerable: true, get: function () { return catalog_pipeline_1.unreachablePredicateKind; } });
|
|
120
|
+
Object.defineProperty(exports, "unreachableRenameUnnamed", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableRenameUnnamed; } });
|
|
111
121
|
Object.defineProperty(exports, "validateWorkflow", { enumerable: true, get: function () { return catalog_pipeline_1.validateWorkflow; } });
|
|
112
122
|
Object.defineProperty(exports, "WORKFLOW_BRANCH_LABELS", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_BRANCH_LABELS; } });
|
|
113
123
|
Object.defineProperty(exports, "WORKFLOW_CALL_CONTRACT", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_CALL_CONTRACT; } });
|
|
@@ -125,17 +135,23 @@ Object.defineProperty(exports, "WORKFLOW_NODE_ID_PATTERN", { enumerable: true, g
|
|
|
125
135
|
Object.defineProperty(exports, "WORKFLOW_NODE_KINDS", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_NODE_KINDS; } });
|
|
126
136
|
Object.defineProperty(exports, "WORKFLOW_NODE_WIDTH", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_NODE_WIDTH; } });
|
|
127
137
|
Object.defineProperty(exports, "WORKFLOW_PREDICATE_KINDS", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_PREDICATE_KINDS; } });
|
|
138
|
+
Object.defineProperty(exports, "WORKFLOW_RENAME_MAX_COLUMNS", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_RENAME_MAX_COLUMNS; } });
|
|
139
|
+
Object.defineProperty(exports, "WORKFLOW_RENAME_UNNAMED", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_RENAME_UNNAMED; } });
|
|
128
140
|
Object.defineProperty(exports, "WORKFLOW_ROW_GAP", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_ROW_GAP; } });
|
|
129
141
|
Object.defineProperty(exports, "WORKFLOW_SKIP_REASONS", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_SKIP_REASONS; } });
|
|
130
142
|
Object.defineProperty(exports, "WORKFLOW_STATUSES", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_STATUSES; } });
|
|
131
143
|
Object.defineProperty(exports, "workflowColumnX", { enumerable: true, get: function () { return catalog_pipeline_1.workflowColumnX; } });
|
|
132
144
|
Object.defineProperty(exports, "workflowRowY", { enumerable: true, get: function () { return catalog_pipeline_1.workflowRowY; } });
|
|
133
145
|
Object.defineProperty(exports, "workflowCallMode", { enumerable: true, get: function () { return catalog_pipeline_1.workflowCallMode; } });
|
|
146
|
+
Object.defineProperty(exports, "workflowFilterColumns", { enumerable: true, get: function () { return catalog_pipeline_1.workflowFilterColumns; } });
|
|
134
147
|
Object.defineProperty(exports, "workflowFilterMatches", { enumerable: true, get: function () { return catalog_pipeline_1.workflowFilterMatches; } });
|
|
135
148
|
Object.defineProperty(exports, "workflowGraphHash", { enumerable: true, get: function () { return catalog_pipeline_1.workflowGraphHash; } });
|
|
149
|
+
Object.defineProperty(exports, "workflowKnownColumns", { enumerable: true, get: function () { return catalog_pipeline_1.workflowKnownColumns; } });
|
|
136
150
|
Object.defineProperty(exports, "workflowNarrowedTypes", { enumerable: true, get: function () { return catalog_pipeline_1.workflowNarrowedTypes; } });
|
|
137
151
|
Object.defineProperty(exports, "workflowNodeRuns", { enumerable: true, get: function () { return catalog_pipeline_1.workflowNodeRuns; } });
|
|
138
152
|
Object.defineProperty(exports, "workflowRunOrder", { enumerable: true, get: function () { return catalog_pipeline_1.workflowRunOrder; } });
|
|
153
|
+
Object.defineProperty(exports, "renameColumnRefusals", { enumerable: true, get: function () { return catalog_pipeline_1.renameColumnRefusals; } });
|
|
154
|
+
Object.defineProperty(exports, "workflowRenameUnnamed", { enumerable: true, get: function () { return catalog_pipeline_1.workflowRenameUnnamed; } });
|
|
139
155
|
// How a staged batch is written down. Exported because `CatalogStageStore` is a
|
|
140
156
|
// seam a host can implement — a stage kept in object storage or a columnar
|
|
141
157
|
// warehouse rather than the catalog database is the case the interface exists
|
|
@@ -149,6 +165,7 @@ Object.defineProperty(exports, "classifyStagePayload", { enumerable: true, get:
|
|
|
149
165
|
Object.defineProperty(exports, "decodeStageRows", { enumerable: true, get: function () { return catalog_stage_encoding_1.decodeStageRows; } });
|
|
150
166
|
Object.defineProperty(exports, "encodeStageRows", { enumerable: true, get: function () { return catalog_stage_encoding_1.encodeStageRows; } });
|
|
151
167
|
Object.defineProperty(exports, "isColumnarStageBatch", { enumerable: true, get: function () { return catalog_stage_encoding_1.isColumnarStageBatch; } });
|
|
168
|
+
Object.defineProperty(exports, "renameStagePayload", { enumerable: true, get: function () { return catalog_stage_encoding_1.renameStagePayload; } });
|
|
152
169
|
// The environment surface: which catalog database a call is served from, and
|
|
153
170
|
// how a connector or a transform is promoted between them.
|
|
154
171
|
__exportStar(require("./catalog.environment"), exports);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type CatalogCodeContext, type CatalogTransform, type TransformLanguage, type TransformResult, type TransformRunner } from './catalog.pipeline';
|
|
1
|
+
import { type CatalogCodeContext, type CatalogTransform, type TransformLanguage, type TransformResult, type TransformRunner, type TransformStream } from './catalog.pipeline';
|
|
2
2
|
export interface TransformRunnerOptions {
|
|
3
3
|
/**
|
|
4
4
|
* A Python virtualenv whose interpreter runs transforms.
|
|
@@ -88,6 +88,47 @@ export declare class SubprocessTransformRunner implements TransformRunner {
|
|
|
88
88
|
timeoutMs?: number;
|
|
89
89
|
context?: CatalogCodeContext;
|
|
90
90
|
}): Promise<TransformResult>;
|
|
91
|
+
/**
|
|
92
|
+
* Run a `'record'`-mode transform over a stream, and hand the rows back as a
|
|
93
|
+
* stream.
|
|
94
|
+
*
|
|
95
|
+
* ## What is and is not different from {@link run}
|
|
96
|
+
*
|
|
97
|
+
* **The isolation is not different, at all.** Same interpreter, same
|
|
98
|
+
* {@link CHILD_PROCESS_OPTIONS} — the same `{PATH, NODE_ENV}`, the same
|
|
99
|
+
* temporary cwd, the same process group — and the context still travels beside
|
|
100
|
+
* the records on stdin rather than in the environment. It is worth being blunt
|
|
101
|
+
* about why that is easy to say: the child here is **not longer-lived than the
|
|
102
|
+
* one `run` spawns.** Both live for exactly one node run and are gone when it
|
|
103
|
+
* ends. `run` was never a spawn per batch; it was a spawn per node with the
|
|
104
|
+
* whole dataset in one blob. So there is no new window in which state could
|
|
105
|
+
* leak between batches or between runs, because there was never a window to
|
|
106
|
+
* widen. What a transform may retain *within* one run is stated exactly on
|
|
107
|
+
* {@link javascriptRecordHarness}.
|
|
108
|
+
*
|
|
109
|
+
* **The timeout is different, and it has to be.** See {@link stallError}.
|
|
110
|
+
*
|
|
111
|
+
* **Failure names a record.** A batch call can only report that the transform
|
|
112
|
+
* threw; here the child counts what it has consumed and puts that number on
|
|
113
|
+
* every line, so a stream that dies names the record it died on. What was
|
|
114
|
+
* already staged is a matter for the caller — for the connector runner it sits
|
|
115
|
+
* in an uncommitted snapshot, for a workflow node it is overwritten by the next
|
|
116
|
+
* attempt — and in neither case does a watermark move, because nothing here
|
|
117
|
+
* reaches a commit.
|
|
118
|
+
*
|
|
119
|
+
* ## The one shape that would deadlock, and how it is avoided
|
|
120
|
+
*
|
|
121
|
+
* The writer runs as a floating loop and the reader is driven by the consumer
|
|
122
|
+
* pulling rows. Awaiting the writer before yielding the first row is the one
|
|
123
|
+
* arrangement that hangs: the child fills its stdout buffer with rows nobody is
|
|
124
|
+
* draining, stops reading stdin, and this side waits forever for a `drain` that
|
|
125
|
+
* requires the reader that has not started. Written the way it is, a slow
|
|
126
|
+
* consumer simply back-pressures the whole chain, which is the point.
|
|
127
|
+
*/
|
|
128
|
+
runStream(transform: Pick<CatalogTransform, 'language' | 'code' | 'mode'>, records: AsyncIterable<unknown>, options?: {
|
|
129
|
+
timeoutMs?: number;
|
|
130
|
+
context?: CatalogCodeContext;
|
|
131
|
+
}): Promise<TransformStream>;
|
|
91
132
|
private execute;
|
|
92
133
|
private spawn;
|
|
93
134
|
/** Cached, including the negative answer — probing on every run is wasteful. */
|