@dudousxd/nestjs-catalog 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.environment.d.ts +64 -1
- package/dist/catalog.environment.js +42 -3
- package/dist/catalog.pipeline.d.ts +252 -0
- package/dist/catalog.pipeline.js +133 -1
- package/dist/client.d.ts +2 -2
- package/dist/client.js +13 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +10 -4
- package/dist/transform-runner.d.ts +42 -1
- package/dist/transform-runner.js +677 -28
- package/package.json +1 -1
package/dist/client.js
CHANGED
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* types are.
|
|
12
12
|
*/
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
-
exports.
|
|
15
|
-
exports.DELETE_RECONCILIATION_STRATEGIES = exports.callableWorkflowBlock = exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.unreachableRenameUnnamed = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableFilterPredicateKind = exports.unreachableFilterOperator = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowKnownColumns = exports.workflowRenameUnnamed = exports.renameColumnRefusals = exports.workflowFilterMatches = exports.workflowFilterColumns = exports.isWorkflowSkipReason = exports.isWorkflowRenameUnnamed = exports.isWorkflowRenameColumns = exports.isWorkflowPredicateKind = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowBranchLabel = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_RENAME_UNNAMED = exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.workflowRowY = void 0;
|
|
14
|
+
exports.WORKFLOW_COLUMN_GAP = exports.describeVersionPin = exports.describeLiveVersion = exports.unreachableReusableNodeKind = exports.nodeKindIsReusable = exports.NODE_KIND_IS_REUSABLE = exports.REUSABLE_NODE_KINDS = exports.isReusableNodeKind = exports.isReusableNodeBody = exports.reusableNodeBodyOf = exports.applyReusableNode = exports.workflowCallMode = exports.unreachableCallMode = exports.isWorkflowCallMode = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_CALL_CONTRACT = exports.readWorkflowCallOutput = exports.TRANSFORM_LANGUAGES = exports.REDACTED_SECRET = exports.liveWorkflowVersion = exports.isWorkflowNode = exports.isWorkflowEdge = exports.SOURCE_FORMATS = exports.recordModeRefusal = exports.transformMode = exports.TRANSFORM_MODES = exports.isTransformMode = exports.isTransformLanguage = exports.isSourceFormat = exports.isConnectorKind = exports.CONNECTOR_KINDS = exports.transformShape = exports.transformDeclaresModule = exports.catalogRoutes = exports.UnsafeIdentifierError = exports.physicalColumn = exports.outputAlias = exports.isSafeIdentifier = exports.VALUELESS_FILTER_OPERATORS = exports.resolveObjectFilters = exports.parseObjectFilter = exports.offeredFilterOperators = exports.isCatalogFilterOperator = exports.filterOperatorsFor = exports.filterOperatorTakesValue = exports.encodeObjectFilter = exports.coerceFilterValue = exports.CATALOG_FILTER_OPERATORS = exports.CATALOG_FILTER_LIMIT = exports.CATALOG_REVISION_LIMIT = void 0;
|
|
15
|
+
exports.DELETE_RECONCILIATION_STRATEGIES = exports.callableWorkflowBlock = exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.unreachableRenameUnnamed = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableFilterPredicateKind = exports.unreachableFilterOperator = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowKnownColumns = exports.workflowRenameUnnamed = exports.renameColumnRefusals = exports.workflowFilterMatches = exports.workflowFilterColumns = exports.isWorkflowSkipReason = exports.isWorkflowRenameUnnamed = exports.isWorkflowRenameColumns = exports.isWorkflowPredicateKind = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowBranchLabel = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_RENAME_UNNAMED = exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.workflowRowY = exports.workflowColumnX = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_HEIGHT = void 0;
|
|
16
16
|
exports.isDeleteReconciliationStrategy = isDeleteReconciliationStrategy;
|
|
17
17
|
exports.pipelineExpectationRoutes = pipelineExpectationRoutes;
|
|
18
18
|
// A value, not a type: a screen saying how far back the history goes should read
|
|
@@ -144,6 +144,17 @@ Object.defineProperty(exports, "CONNECTOR_KINDS", { enumerable: true, get: funct
|
|
|
144
144
|
Object.defineProperty(exports, "isConnectorKind", { enumerable: true, get: function () { return catalog_pipeline_1.isConnectorKind; } });
|
|
145
145
|
Object.defineProperty(exports, "isSourceFormat", { enumerable: true, get: function () { return catalog_pipeline_1.isSourceFormat; } });
|
|
146
146
|
Object.defineProperty(exports, "isTransformLanguage", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformLanguage; } });
|
|
147
|
+
// The editor offers exactly the modes the runner can call, and applies the
|
|
148
|
+
// default through the same function the store and both runners do. A screen
|
|
149
|
+
// carrying its own `?? 'batch'` is the second copy that decides a stored
|
|
150
|
+
// transform means something else.
|
|
151
|
+
Object.defineProperty(exports, "isTransformMode", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformMode; } });
|
|
152
|
+
Object.defineProperty(exports, "TRANSFORM_MODES", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_MODES; } });
|
|
153
|
+
Object.defineProperty(exports, "transformMode", { enumerable: true, get: function () { return catalog_pipeline_1.transformMode; } });
|
|
154
|
+
// Whether the two impossible combinations are impossible, asked while the
|
|
155
|
+
// author is still looking at the code rather than at three in the morning when
|
|
156
|
+
// a schedule fires.
|
|
157
|
+
Object.defineProperty(exports, "recordModeRefusal", { enumerable: true, get: function () { return catalog_pipeline_1.recordModeRefusal; } });
|
|
147
158
|
Object.defineProperty(exports, "SOURCE_FORMATS", { enumerable: true, get: function () { return catalog_pipeline_1.SOURCE_FORMATS; } });
|
|
148
159
|
// The canvas narrows nodes and edges it reads back from HTTP. Without these
|
|
149
160
|
// it either imports them from the package root — dragging NestJS and MikroORM
|
package/dist/index.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export { type CatalogOverlayStore, FileCatalogOverlayStore, InMemoryCatalogOverl
|
|
|
8
8
|
export { CATALOG_OVERLAY_STORE } from './catalog.overlay-store.token';
|
|
9
9
|
export { MikroOrmCatalogRegistry } from './catalog.registry';
|
|
10
10
|
export { CatalogRegistry } from './catalog.registry.base';
|
|
11
|
-
export { CATALOG_PIPELINE_STORE, CODE_CONTEXT_CONTRACT, type CatalogCodeContext, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogLoadExpectations, type CatalogLoadExpectationStore, type CatalogPipelineStore, type CatalogStageStore, type CatalogTransform, type CatalogTransformFunction, type CatalogTransformInput, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowRelease, type CatalogWorkflowReleaseStore, type CatalogWorkflowStore, type CallableWorkflowBlock, type CallableWorkflowDisagreement, type CallableWorkflowRef, callableWorkflowBlock, type ConnectorKind, type ConnectorRun, type DeleteReconciliation, isConnectorKind, isPipelineStore, isSourceFormat, isTransformLanguage, type LoadExpectation, type RowCountBound, SOURCE_FORMATS, type SourceFormat, unreachableSourceFormat, type StoredLoadExpectation, readWorkflowCallOutput, REDACTED_SECRET, supportsLoadExpectations, supportsReusableNodes, supportsTransformPins, supportsTransformRevisions, applyReusableNode, type CatalogReusableNode, type CatalogReusableNodeStore, type CatalogReusableNodeUse, describeLiveVersion, describeVersionPin, isReusableNodeBody, isReusableNodeKind, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, REUSABLE_NODE_KINDS, type ReusableNodeBody, type ReusableNodeKind, type ReusableNodeRef, type ReusableSinkBody, type ReusableSourceBody, reusableNodeBodyOf, unreachableReusableNodeKind, type VersionPinCopy, isWorkflowBranchLabel, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowNode, isWorkflowCallMode, isWorkflowNodeKind, isWorkflowPredicateKind, isWorkflowRenameColumns, isWorkflowRenameUnnamed, isWorkflowSkipReason, isWorkflowStatus, liveWorkflowVersion, supportsWorkflowReleases, supportsWorkflows, supportsWorkflowStages, supportsStagePayloads, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, type TransformLanguage, type TransformResult, type TransformRunner, unreachableFilterOperator, unreachableFilterPredicateKind, unreachableCallMode, unreachableNodeKind, unreachablePredicateKind, unreachableRenameUnnamed, validateWorkflow, WORKFLOW_BRANCH_LABELS, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, WORKFLOW_COLUMN_GAP, WORKFLOW_EXECUTION_MODES, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_HEIGHT, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_NODE_WIDTH, WORKFLOW_PREDICATE_KINDS, WORKFLOW_RENAME_MAX_COLUMNS, WORKFLOW_RENAME_UNNAMED, WORKFLOW_ROW_GAP, WORKFLOW_SKIP_REASONS, WORKFLOW_STATUSES, type WorkflowBranchLabel, workflowColumnX, workflowRowY, type WorkflowCallEnvelope, type WorkflowCallMode, workflowCallMode, type WorkflowCallNode, type WorkflowCallOutput, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowFilterAll, type WorkflowFilterAny, type WorkflowFilterComparison, type WorkflowFilterGroup, workflowFilterColumns, workflowFilterMatches, type WorkflowFilterNode, type WorkflowFilterOneOf, type WorkflowFilterOperator, type WorkflowFilterPredicate, type WorkflowFilterPredicateKind, type WorkflowFilterPresence, type WorkflowFilterValue, type WorkflowGraph, workflowGraphHash, type WorkflowEnvPredicate, type WorkflowIfNode, type WorkflowIfPredicate, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, workflowKnownColumns, workflowNarrowedTypes, type WorkflowNodeOutcome, workflowNodeRuns, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowRunOrderEntry, type WorkflowPredicateKind, renameColumnRefusals, type WorkflowRenameNode, type WorkflowRenameUnnamed, workflowRenameUnnamed, type WorkflowRowCountPredicate, type WorkflowSinkNode, type WorkflowSkipReason, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowStatus, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
|
|
11
|
+
export { CATALOG_PIPELINE_STORE, CODE_CONTEXT_CONTRACT, type CatalogCodeContext, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogLoadExpectations, type CatalogLoadExpectationStore, type CatalogPipelineStore, type CatalogStageStore, type CatalogRecordTransformFunction, type CatalogRecordTransformInput, type CatalogTransform, type CatalogTransformFunction, type CatalogTransformInput, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowRelease, type CatalogWorkflowReleaseStore, type CatalogWorkflowStore, type CallableWorkflowBlock, type CallableWorkflowDisagreement, type CallableWorkflowRef, callableWorkflowBlock, type ConnectorKind, type ConnectorRun, type DeleteReconciliation, isConnectorKind, isPipelineStore, isSourceFormat, isTransformLanguage, isTransformMode, type LoadExpectation, type RowCountBound, SOURCE_FORMATS, type SourceFormat, unreachableSourceFormat, type StoredLoadExpectation, readWorkflowCallOutput, REDACTED_SECRET, supportsLoadExpectations, supportsReusableNodes, supportsTransformPins, supportsTransformRevisions, supportsTransformStreaming, applyReusableNode, type CatalogReusableNode, type CatalogReusableNodeStore, type CatalogReusableNodeUse, describeLiveVersion, describeVersionPin, isReusableNodeBody, isReusableNodeKind, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, REUSABLE_NODE_KINDS, type ReusableNodeBody, type ReusableNodeKind, type ReusableNodeRef, type ReusableSinkBody, type ReusableSourceBody, reusableNodeBodyOf, unreachableReusableNodeKind, type VersionPinCopy, isWorkflowBranchLabel, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowNode, isWorkflowCallMode, isWorkflowNodeKind, isWorkflowPredicateKind, isWorkflowRenameColumns, isWorkflowRenameUnnamed, isWorkflowSkipReason, isWorkflowStatus, liveWorkflowVersion, supportsWorkflowReleases, supportsWorkflows, supportsWorkflowStages, supportsStagePayloads, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, TRANSFORM_MODES, recordModeRefusal, transformMode, type TransformLanguage, type TransformMode, type TransformResult, type TransformRunner, type TransformStream, type TransformStreamSummary, unreachableFilterOperator, unreachableTransformMode, unreachableFilterPredicateKind, unreachableCallMode, unreachableNodeKind, unreachablePredicateKind, unreachableRenameUnnamed, validateWorkflow, WORKFLOW_BRANCH_LABELS, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, WORKFLOW_COLUMN_GAP, WORKFLOW_EXECUTION_MODES, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_HEIGHT, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_NODE_WIDTH, WORKFLOW_PREDICATE_KINDS, WORKFLOW_RENAME_MAX_COLUMNS, WORKFLOW_RENAME_UNNAMED, WORKFLOW_ROW_GAP, WORKFLOW_SKIP_REASONS, WORKFLOW_STATUSES, type WorkflowBranchLabel, workflowColumnX, workflowRowY, type WorkflowCallEnvelope, type WorkflowCallMode, workflowCallMode, type WorkflowCallNode, type WorkflowCallOutput, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowFilterAll, type WorkflowFilterAny, type WorkflowFilterComparison, type WorkflowFilterGroup, workflowFilterColumns, workflowFilterMatches, type WorkflowFilterNode, type WorkflowFilterOneOf, type WorkflowFilterOperator, type WorkflowFilterPredicate, type WorkflowFilterPredicateKind, type WorkflowFilterPresence, type WorkflowFilterValue, type WorkflowGraph, workflowGraphHash, type WorkflowEnvPredicate, type WorkflowIfNode, type WorkflowIfPredicate, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, workflowKnownColumns, workflowNarrowedTypes, type WorkflowNodeOutcome, workflowNodeRuns, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowRunOrderEntry, type WorkflowPredicateKind, renameColumnRefusals, type WorkflowRenameNode, type WorkflowRenameUnnamed, workflowRenameUnnamed, type WorkflowRowCountPredicate, type WorkflowSinkNode, type WorkflowSkipReason, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowStatus, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
|
|
12
12
|
export { type ColumnarStageBatch, STAGE_ENCODING, STAGE_ENCODING_VERSION, type StagePayload, classifyStagePayload, decodeStageRows, encodeStageRows, isColumnarStageBatch, renameStagePayload, type StageRenamePlan, type StageRenameResult, } from './catalog.stage-encoding';
|
|
13
13
|
export * from './catalog.environment';
|
|
14
14
|
export { QueryCache } from './catalog.query-cache';
|
package/dist/index.js
CHANGED
|
@@ -14,10 +14,10 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.
|
|
18
|
-
exports.
|
|
19
|
-
exports.
|
|
20
|
-
exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.UnsafeIdentifierError = exports.supportsCarryForward = exports.physicalColumn = exports.outputAlias = exports.isWriteStore = exports.isSafeIdentifier = exports.isReservedColumn = exports.isCatalogStoreCapabilities = exports.findColumnCollisions = exports.supportsObjectFilters = exports.CatalogColumnCollisionError = exports.CATALOG_STORE = exports.CATALOG_SNAPSHOT_MODES = exports.CATALOG_RESERVED_COLUMNS = exports.assertSafeIdentifier = exports.assertNoColumnCollisions = exports.StaticKeyPrincipalResolver = exports.readableObjectPage = exports.mayWrite = exports.mayRead = exports.maySeeClassification = exports.PRINCIPAL_ACTOR_SEPARATOR = void 0;
|
|
17
|
+
exports.isWorkflowBranchLabel = exports.unreachableReusableNodeKind = exports.reusableNodeBodyOf = exports.REUSABLE_NODE_KINDS = exports.nodeKindIsReusable = exports.NODE_KIND_IS_REUSABLE = exports.isReusableNodeKind = exports.isReusableNodeBody = exports.describeVersionPin = exports.describeLiveVersion = exports.applyReusableNode = exports.supportsTransformStreaming = exports.supportsTransformRevisions = exports.supportsTransformPins = exports.supportsReusableNodes = exports.supportsLoadExpectations = exports.REDACTED_SECRET = exports.readWorkflowCallOutput = exports.unreachableSourceFormat = exports.SOURCE_FORMATS = exports.isTransformMode = exports.isTransformLanguage = exports.isSourceFormat = exports.isPipelineStore = exports.isConnectorKind = exports.callableWorkflowBlock = exports.CONNECTOR_KINDS = exports.CODE_CONTEXT_CONTRACT = exports.CATALOG_PIPELINE_STORE = exports.CatalogRegistry = exports.MikroOrmCatalogRegistry = exports.CATALOG_OVERLAY_STORE = exports.InMemoryCatalogOverlayStore = exports.FileCatalogOverlayStore = exports.CATALOG_OPTIONS = exports.isStreamingQueryStore = exports.isQueryStore = exports.assertReadOnlyShape = exports.CatalogModule = exports.emitCatalog = exports.curationActor = exports.channelNameFor = exports.catalogEventPhase = exports.UNATTRIBUTED_PRINCIPAL_ID = exports.CATALOG_LIB = exports.CATALOG_EVENTS = exports.CATALOG_EVENT_PHASE_FALLBACK = exports.CATALOG_EVENT_PHASE = exports.CatalogType = exports.CatalogProperty = void 0;
|
|
18
|
+
exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_BRANCH_LABELS = exports.validateWorkflow = exports.unreachableRenameUnnamed = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableCallMode = exports.unreachableFilterPredicateKind = exports.unreachableTransformMode = exports.unreachableFilterOperator = exports.transformMode = exports.recordModeRefusal = exports.TRANSFORM_MODES = exports.TRANSFORM_LANGUAGES = exports.TRANSFORM_RUNNER = exports.supportsStagePayloads = exports.supportsWorkflowStages = exports.supportsWorkflows = exports.supportsWorkflowReleases = exports.liveWorkflowVersion = exports.isWorkflowStatus = exports.isWorkflowSkipReason = exports.isWorkflowRenameUnnamed = exports.isWorkflowRenameColumns = exports.isWorkflowPredicateKind = exports.isWorkflowNodeKind = exports.isWorkflowCallMode = exports.isWorkflowNode = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowExecutionMode = exports.isWorkflowEdge = void 0;
|
|
19
|
+
exports.traceOutcomeFilter = exports.supportsSavedQueryRevisions = exports.isWorkspaceStore = exports.isTraceStore = exports.isCatalogTraceOutcome = exports.embeddedVisualization = exports.CATALOG_WORKSPACE_STORE = exports.CATALOG_TRACE_STORE = exports.CATALOG_TRACE_OUTCOMES = exports.CATALOG_REVISION_LIMIT = exports.visibleToPrincipal = exports.searchCatalog = exports.maySearch = exports.emptySearch = exports.bestMatch = exports.MAX_SEARCH_LIMIT = exports.DEFAULT_SEARCH_LIMIT = exports.CatalogService = exports.transformShapeHint = exports.transformShape = exports.transformDeclaresModule = exports.SubprocessTransformRunner = exports.toCsv = exports.guardFormula = exports.csvLines = exports.csvCell = exports.QueryCache = exports.renameStagePayload = exports.isColumnarStageBatch = exports.encodeStageRows = exports.decodeStageRows = exports.classifyStagePayload = exports.STAGE_ENCODING_VERSION = exports.STAGE_ENCODING = exports.workflowRenameUnnamed = exports.renameColumnRefusals = exports.workflowRunOrder = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowKnownColumns = exports.workflowGraphHash = exports.workflowFilterMatches = exports.workflowFilterColumns = exports.workflowCallMode = exports.workflowRowY = exports.workflowColumnX = exports.WORKFLOW_STATUSES = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_RENAME_UNNAMED = void 0;
|
|
20
|
+
exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.UnsafeIdentifierError = exports.supportsCarryForward = exports.physicalColumn = exports.outputAlias = exports.isWriteStore = exports.isSafeIdentifier = exports.isReservedColumn = exports.isCatalogStoreCapabilities = exports.findColumnCollisions = exports.supportsObjectFilters = exports.CatalogColumnCollisionError = exports.CATALOG_STORE = exports.CATALOG_SNAPSHOT_MODES = exports.CATALOG_RESERVED_COLUMNS = exports.assertSafeIdentifier = exports.assertNoColumnCollisions = exports.StaticKeyPrincipalResolver = exports.readableObjectPage = exports.mayWrite = exports.mayRead = exports.maySeeClassification = exports.PRINCIPAL_ACTOR_SEPARATOR = exports.parsePrincipalId = exports.hasScope = exports.expandScopes = exports.delegatePrincipal = exports.composePrincipalId = exports.CATALOG_PRINCIPAL_RESOLVER = void 0;
|
|
21
21
|
var catalog_decorators_1 = require("./catalog.decorators");
|
|
22
22
|
Object.defineProperty(exports, "CatalogProperty", { enumerable: true, get: function () { return catalog_decorators_1.CatalogProperty; } });
|
|
23
23
|
Object.defineProperty(exports, "CatalogType", { enumerable: true, get: function () { return catalog_decorators_1.CatalogType; } });
|
|
@@ -65,6 +65,7 @@ Object.defineProperty(exports, "isConnectorKind", { enumerable: true, get: funct
|
|
|
65
65
|
Object.defineProperty(exports, "isPipelineStore", { enumerable: true, get: function () { return catalog_pipeline_1.isPipelineStore; } });
|
|
66
66
|
Object.defineProperty(exports, "isSourceFormat", { enumerable: true, get: function () { return catalog_pipeline_1.isSourceFormat; } });
|
|
67
67
|
Object.defineProperty(exports, "isTransformLanguage", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformLanguage; } });
|
|
68
|
+
Object.defineProperty(exports, "isTransformMode", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformMode; } });
|
|
68
69
|
Object.defineProperty(exports, "SOURCE_FORMATS", { enumerable: true, get: function () { return catalog_pipeline_1.SOURCE_FORMATS; } });
|
|
69
70
|
Object.defineProperty(exports, "unreachableSourceFormat", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableSourceFormat; } });
|
|
70
71
|
Object.defineProperty(exports, "readWorkflowCallOutput", { enumerable: true, get: function () { return catalog_pipeline_1.readWorkflowCallOutput; } });
|
|
@@ -73,6 +74,7 @@ Object.defineProperty(exports, "supportsLoadExpectations", { enumerable: true, g
|
|
|
73
74
|
Object.defineProperty(exports, "supportsReusableNodes", { enumerable: true, get: function () { return catalog_pipeline_1.supportsReusableNodes; } });
|
|
74
75
|
Object.defineProperty(exports, "supportsTransformPins", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformPins; } });
|
|
75
76
|
Object.defineProperty(exports, "supportsTransformRevisions", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformRevisions; } });
|
|
77
|
+
Object.defineProperty(exports, "supportsTransformStreaming", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformStreaming; } });
|
|
76
78
|
Object.defineProperty(exports, "applyReusableNode", { enumerable: true, get: function () { return catalog_pipeline_1.applyReusableNode; } });
|
|
77
79
|
Object.defineProperty(exports, "describeLiveVersion", { enumerable: true, get: function () { return catalog_pipeline_1.describeLiveVersion; } });
|
|
78
80
|
Object.defineProperty(exports, "describeVersionPin", { enumerable: true, get: function () { return catalog_pipeline_1.describeVersionPin; } });
|
|
@@ -106,7 +108,11 @@ Object.defineProperty(exports, "supportsWorkflowStages", { enumerable: true, get
|
|
|
106
108
|
Object.defineProperty(exports, "supportsStagePayloads", { enumerable: true, get: function () { return catalog_pipeline_1.supportsStagePayloads; } });
|
|
107
109
|
Object.defineProperty(exports, "TRANSFORM_RUNNER", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_RUNNER; } });
|
|
108
110
|
Object.defineProperty(exports, "TRANSFORM_LANGUAGES", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_LANGUAGES; } });
|
|
111
|
+
Object.defineProperty(exports, "TRANSFORM_MODES", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_MODES; } });
|
|
112
|
+
Object.defineProperty(exports, "recordModeRefusal", { enumerable: true, get: function () { return catalog_pipeline_1.recordModeRefusal; } });
|
|
113
|
+
Object.defineProperty(exports, "transformMode", { enumerable: true, get: function () { return catalog_pipeline_1.transformMode; } });
|
|
109
114
|
Object.defineProperty(exports, "unreachableFilterOperator", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableFilterOperator; } });
|
|
115
|
+
Object.defineProperty(exports, "unreachableTransformMode", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableTransformMode; } });
|
|
110
116
|
Object.defineProperty(exports, "unreachableFilterPredicateKind", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableFilterPredicateKind; } });
|
|
111
117
|
Object.defineProperty(exports, "unreachableCallMode", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableCallMode; } });
|
|
112
118
|
Object.defineProperty(exports, "unreachableNodeKind", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableNodeKind; } });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type CatalogCodeContext, type CatalogTransform, type TransformLanguage, type TransformResult, type TransformRunner } from './catalog.pipeline';
|
|
1
|
+
import { type CatalogCodeContext, type CatalogTransform, type TransformLanguage, type TransformResult, type TransformRunner, type TransformStream } from './catalog.pipeline';
|
|
2
2
|
export interface TransformRunnerOptions {
|
|
3
3
|
/**
|
|
4
4
|
* A Python virtualenv whose interpreter runs transforms.
|
|
@@ -88,6 +88,47 @@ export declare class SubprocessTransformRunner implements TransformRunner {
|
|
|
88
88
|
timeoutMs?: number;
|
|
89
89
|
context?: CatalogCodeContext;
|
|
90
90
|
}): Promise<TransformResult>;
|
|
91
|
+
/**
|
|
92
|
+
* Run a `'record'`-mode transform over a stream, and hand the rows back as a
|
|
93
|
+
* stream.
|
|
94
|
+
*
|
|
95
|
+
* ## What is and is not different from {@link run}
|
|
96
|
+
*
|
|
97
|
+
* **The isolation is not different, at all.** Same interpreter, same
|
|
98
|
+
* {@link CHILD_PROCESS_OPTIONS} — the same `{PATH, NODE_ENV}`, the same
|
|
99
|
+
* temporary cwd, the same process group — and the context still travels beside
|
|
100
|
+
* the records on stdin rather than in the environment. It is worth being blunt
|
|
101
|
+
* about why that is easy to say: the child here is **not longer-lived than the
|
|
102
|
+
* one `run` spawns.** Both live for exactly one node run and are gone when it
|
|
103
|
+
* ends. `run` was never a spawn per batch; it was a spawn per node with the
|
|
104
|
+
* whole dataset in one blob. So there is no new window in which state could
|
|
105
|
+
* leak between batches or between runs, because there was never a window to
|
|
106
|
+
* widen. What a transform may retain *within* one run is stated exactly on
|
|
107
|
+
* {@link javascriptRecordHarness}.
|
|
108
|
+
*
|
|
109
|
+
* **The timeout is different, and it has to be.** See {@link stallError}.
|
|
110
|
+
*
|
|
111
|
+
* **Failure names a record.** A batch call can only report that the transform
|
|
112
|
+
* threw; here the child counts what it has consumed and puts that number on
|
|
113
|
+
* every line, so a stream that dies names the record it died on. What was
|
|
114
|
+
* already staged is a matter for the caller — for the connector runner it sits
|
|
115
|
+
* in an uncommitted snapshot, for a workflow node it is overwritten by the next
|
|
116
|
+
* attempt — and in neither case does a watermark move, because nothing here
|
|
117
|
+
* reaches a commit.
|
|
118
|
+
*
|
|
119
|
+
* ## The one shape that would deadlock, and how it is avoided
|
|
120
|
+
*
|
|
121
|
+
* The writer runs as a floating loop and the reader is driven by the consumer
|
|
122
|
+
* pulling rows. Awaiting the writer before yielding the first row is the one
|
|
123
|
+
* arrangement that hangs: the child fills its stdout buffer with rows nobody is
|
|
124
|
+
* draining, stops reading stdin, and this side waits forever for a `drain` that
|
|
125
|
+
* requires the reader that has not started. Written the way it is, a slow
|
|
126
|
+
* consumer simply back-pressures the whole chain, which is the point.
|
|
127
|
+
*/
|
|
128
|
+
runStream(transform: Pick<CatalogTransform, 'language' | 'code' | 'mode'>, records: AsyncIterable<unknown>, options?: {
|
|
129
|
+
timeoutMs?: number;
|
|
130
|
+
context?: CatalogCodeContext;
|
|
131
|
+
}): Promise<TransformStream>;
|
|
91
132
|
private execute;
|
|
92
133
|
private spawn;
|
|
93
134
|
/** Cached, including the negative answer — probing on every run is wasteful. */
|