@dudousxd/nestjs-catalog 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.environment.d.ts +12 -1
- package/dist/catalog.environment.js +12 -1
- package/dist/catalog.pipeline.d.ts +252 -0
- package/dist/catalog.pipeline.js +133 -1
- package/dist/client.d.ts +2 -2
- package/dist/client.js +13 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +10 -4
- package/dist/transform-runner.d.ts +42 -1
- package/dist/transform-runner.js +677 -28
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -14,10 +14,10 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.
|
|
18
|
-
exports.
|
|
19
|
-
exports.
|
|
20
|
-
exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.UnsafeIdentifierError = exports.supportsCarryForward = exports.physicalColumn = exports.outputAlias = exports.isWriteStore = exports.isSafeIdentifier = exports.isReservedColumn = exports.isCatalogStoreCapabilities = exports.findColumnCollisions = exports.supportsObjectFilters = exports.CatalogColumnCollisionError = exports.CATALOG_STORE = exports.CATALOG_SNAPSHOT_MODES = exports.CATALOG_RESERVED_COLUMNS = exports.assertSafeIdentifier = exports.assertNoColumnCollisions = exports.StaticKeyPrincipalResolver = exports.readableObjectPage = exports.mayWrite = exports.mayRead = exports.maySeeClassification = exports.PRINCIPAL_ACTOR_SEPARATOR = void 0;
|
|
17
|
+
exports.isWorkflowBranchLabel = exports.unreachableReusableNodeKind = exports.reusableNodeBodyOf = exports.REUSABLE_NODE_KINDS = exports.nodeKindIsReusable = exports.NODE_KIND_IS_REUSABLE = exports.isReusableNodeKind = exports.isReusableNodeBody = exports.describeVersionPin = exports.describeLiveVersion = exports.applyReusableNode = exports.supportsTransformStreaming = exports.supportsTransformRevisions = exports.supportsTransformPins = exports.supportsReusableNodes = exports.supportsLoadExpectations = exports.REDACTED_SECRET = exports.readWorkflowCallOutput = exports.unreachableSourceFormat = exports.SOURCE_FORMATS = exports.isTransformMode = exports.isTransformLanguage = exports.isSourceFormat = exports.isPipelineStore = exports.isConnectorKind = exports.callableWorkflowBlock = exports.CONNECTOR_KINDS = exports.CODE_CONTEXT_CONTRACT = exports.CATALOG_PIPELINE_STORE = exports.CatalogRegistry = exports.MikroOrmCatalogRegistry = exports.CATALOG_OVERLAY_STORE = exports.InMemoryCatalogOverlayStore = exports.FileCatalogOverlayStore = exports.CATALOG_OPTIONS = exports.isStreamingQueryStore = exports.isQueryStore = exports.assertReadOnlyShape = exports.CatalogModule = exports.emitCatalog = exports.curationActor = exports.channelNameFor = exports.catalogEventPhase = exports.UNATTRIBUTED_PRINCIPAL_ID = exports.CATALOG_LIB = exports.CATALOG_EVENTS = exports.CATALOG_EVENT_PHASE_FALLBACK = exports.CATALOG_EVENT_PHASE = exports.CatalogType = exports.CatalogProperty = void 0;
|
|
18
|
+
exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_BRANCH_LABELS = exports.validateWorkflow = exports.unreachableRenameUnnamed = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableCallMode = exports.unreachableFilterPredicateKind = exports.unreachableTransformMode = exports.unreachableFilterOperator = exports.transformMode = exports.recordModeRefusal = exports.TRANSFORM_MODES = exports.TRANSFORM_LANGUAGES = exports.TRANSFORM_RUNNER = exports.supportsStagePayloads = exports.supportsWorkflowStages = exports.supportsWorkflows = exports.supportsWorkflowReleases = exports.liveWorkflowVersion = exports.isWorkflowStatus = exports.isWorkflowSkipReason = exports.isWorkflowRenameUnnamed = exports.isWorkflowRenameColumns = exports.isWorkflowPredicateKind = exports.isWorkflowNodeKind = exports.isWorkflowCallMode = exports.isWorkflowNode = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowExecutionMode = exports.isWorkflowEdge = void 0;
|
|
19
|
+
exports.traceOutcomeFilter = exports.supportsSavedQueryRevisions = exports.isWorkspaceStore = exports.isTraceStore = exports.isCatalogTraceOutcome = exports.embeddedVisualization = exports.CATALOG_WORKSPACE_STORE = exports.CATALOG_TRACE_STORE = exports.CATALOG_TRACE_OUTCOMES = exports.CATALOG_REVISION_LIMIT = exports.visibleToPrincipal = exports.searchCatalog = exports.maySearch = exports.emptySearch = exports.bestMatch = exports.MAX_SEARCH_LIMIT = exports.DEFAULT_SEARCH_LIMIT = exports.CatalogService = exports.transformShapeHint = exports.transformShape = exports.transformDeclaresModule = exports.SubprocessTransformRunner = exports.toCsv = exports.guardFormula = exports.csvLines = exports.csvCell = exports.QueryCache = exports.renameStagePayload = exports.isColumnarStageBatch = exports.encodeStageRows = exports.decodeStageRows = exports.classifyStagePayload = exports.STAGE_ENCODING_VERSION = exports.STAGE_ENCODING = exports.workflowRenameUnnamed = exports.renameColumnRefusals = exports.workflowRunOrder = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowKnownColumns = exports.workflowGraphHash = exports.workflowFilterMatches = exports.workflowFilterColumns = exports.workflowCallMode = exports.workflowRowY = exports.workflowColumnX = exports.WORKFLOW_STATUSES = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_RENAME_UNNAMED = void 0;
|
|
20
|
+
exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.UnsafeIdentifierError = exports.supportsCarryForward = exports.physicalColumn = exports.outputAlias = exports.isWriteStore = exports.isSafeIdentifier = exports.isReservedColumn = exports.isCatalogStoreCapabilities = exports.findColumnCollisions = exports.supportsObjectFilters = exports.CatalogColumnCollisionError = exports.CATALOG_STORE = exports.CATALOG_SNAPSHOT_MODES = exports.CATALOG_RESERVED_COLUMNS = exports.assertSafeIdentifier = exports.assertNoColumnCollisions = exports.StaticKeyPrincipalResolver = exports.readableObjectPage = exports.mayWrite = exports.mayRead = exports.maySeeClassification = exports.PRINCIPAL_ACTOR_SEPARATOR = exports.parsePrincipalId = exports.hasScope = exports.expandScopes = exports.delegatePrincipal = exports.composePrincipalId = exports.CATALOG_PRINCIPAL_RESOLVER = void 0;
|
|
21
21
|
var catalog_decorators_1 = require("./catalog.decorators");
|
|
22
22
|
Object.defineProperty(exports, "CatalogProperty", { enumerable: true, get: function () { return catalog_decorators_1.CatalogProperty; } });
|
|
23
23
|
Object.defineProperty(exports, "CatalogType", { enumerable: true, get: function () { return catalog_decorators_1.CatalogType; } });
|
|
@@ -65,6 +65,7 @@ Object.defineProperty(exports, "isConnectorKind", { enumerable: true, get: funct
|
|
|
65
65
|
Object.defineProperty(exports, "isPipelineStore", { enumerable: true, get: function () { return catalog_pipeline_1.isPipelineStore; } });
|
|
66
66
|
Object.defineProperty(exports, "isSourceFormat", { enumerable: true, get: function () { return catalog_pipeline_1.isSourceFormat; } });
|
|
67
67
|
Object.defineProperty(exports, "isTransformLanguage", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformLanguage; } });
|
|
68
|
+
Object.defineProperty(exports, "isTransformMode", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformMode; } });
|
|
68
69
|
Object.defineProperty(exports, "SOURCE_FORMATS", { enumerable: true, get: function () { return catalog_pipeline_1.SOURCE_FORMATS; } });
|
|
69
70
|
Object.defineProperty(exports, "unreachableSourceFormat", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableSourceFormat; } });
|
|
70
71
|
Object.defineProperty(exports, "readWorkflowCallOutput", { enumerable: true, get: function () { return catalog_pipeline_1.readWorkflowCallOutput; } });
|
|
@@ -73,6 +74,7 @@ Object.defineProperty(exports, "supportsLoadExpectations", { enumerable: true, g
|
|
|
73
74
|
Object.defineProperty(exports, "supportsReusableNodes", { enumerable: true, get: function () { return catalog_pipeline_1.supportsReusableNodes; } });
|
|
74
75
|
Object.defineProperty(exports, "supportsTransformPins", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformPins; } });
|
|
75
76
|
Object.defineProperty(exports, "supportsTransformRevisions", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformRevisions; } });
|
|
77
|
+
Object.defineProperty(exports, "supportsTransformStreaming", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformStreaming; } });
|
|
76
78
|
Object.defineProperty(exports, "applyReusableNode", { enumerable: true, get: function () { return catalog_pipeline_1.applyReusableNode; } });
|
|
77
79
|
Object.defineProperty(exports, "describeLiveVersion", { enumerable: true, get: function () { return catalog_pipeline_1.describeLiveVersion; } });
|
|
78
80
|
Object.defineProperty(exports, "describeVersionPin", { enumerable: true, get: function () { return catalog_pipeline_1.describeVersionPin; } });
|
|
@@ -106,7 +108,11 @@ Object.defineProperty(exports, "supportsWorkflowStages", { enumerable: true, get
|
|
|
106
108
|
Object.defineProperty(exports, "supportsStagePayloads", { enumerable: true, get: function () { return catalog_pipeline_1.supportsStagePayloads; } });
|
|
107
109
|
Object.defineProperty(exports, "TRANSFORM_RUNNER", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_RUNNER; } });
|
|
108
110
|
Object.defineProperty(exports, "TRANSFORM_LANGUAGES", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_LANGUAGES; } });
|
|
111
|
+
Object.defineProperty(exports, "TRANSFORM_MODES", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_MODES; } });
|
|
112
|
+
Object.defineProperty(exports, "recordModeRefusal", { enumerable: true, get: function () { return catalog_pipeline_1.recordModeRefusal; } });
|
|
113
|
+
Object.defineProperty(exports, "transformMode", { enumerable: true, get: function () { return catalog_pipeline_1.transformMode; } });
|
|
109
114
|
Object.defineProperty(exports, "unreachableFilterOperator", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableFilterOperator; } });
|
|
115
|
+
Object.defineProperty(exports, "unreachableTransformMode", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableTransformMode; } });
|
|
110
116
|
Object.defineProperty(exports, "unreachableFilterPredicateKind", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableFilterPredicateKind; } });
|
|
111
117
|
Object.defineProperty(exports, "unreachableCallMode", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableCallMode; } });
|
|
112
118
|
Object.defineProperty(exports, "unreachableNodeKind", { enumerable: true, get: function () { return catalog_pipeline_1.unreachableNodeKind; } });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type CatalogCodeContext, type CatalogTransform, type TransformLanguage, type TransformResult, type TransformRunner } from './catalog.pipeline';
|
|
1
|
+
import { type CatalogCodeContext, type CatalogTransform, type TransformLanguage, type TransformResult, type TransformRunner, type TransformStream } from './catalog.pipeline';
|
|
2
2
|
export interface TransformRunnerOptions {
|
|
3
3
|
/**
|
|
4
4
|
* A Python virtualenv whose interpreter runs transforms.
|
|
@@ -88,6 +88,47 @@ export declare class SubprocessTransformRunner implements TransformRunner {
|
|
|
88
88
|
timeoutMs?: number;
|
|
89
89
|
context?: CatalogCodeContext;
|
|
90
90
|
}): Promise<TransformResult>;
|
|
91
|
+
/**
|
|
92
|
+
* Run a `'record'`-mode transform over a stream, and hand the rows back as a
|
|
93
|
+
* stream.
|
|
94
|
+
*
|
|
95
|
+
* ## What is and is not different from {@link run}
|
|
96
|
+
*
|
|
97
|
+
* **The isolation is not different, at all.** Same interpreter, same
|
|
98
|
+
* {@link CHILD_PROCESS_OPTIONS} — the same `{PATH, NODE_ENV}`, the same
|
|
99
|
+
* temporary cwd, the same process group — and the context still travels beside
|
|
100
|
+
* the records on stdin rather than in the environment. It is worth being blunt
|
|
101
|
+
* about why that is easy to say: the child here is **not longer-lived than the
|
|
102
|
+
* one `run` spawns.** Both live for exactly one node run and are gone when it
|
|
103
|
+
* ends. `run` was never a spawn per batch; it was a spawn per node with the
|
|
104
|
+
* whole dataset in one blob. So there is no new window in which state could
|
|
105
|
+
* leak between batches or between runs, because there was never a window to
|
|
106
|
+
* widen. What a transform may retain *within* one run is stated exactly on
|
|
107
|
+
* {@link javascriptRecordHarness}.
|
|
108
|
+
*
|
|
109
|
+
* **The timeout is different, and it has to be.** See {@link stallError}.
|
|
110
|
+
*
|
|
111
|
+
* **Failure names a record.** A batch call can only report that the transform
|
|
112
|
+
* threw; here the child counts what it has consumed and puts that number on
|
|
113
|
+
* every line, so a stream that dies names the record it died on. What was
|
|
114
|
+
* already staged is a matter for the caller — for the connector runner it sits
|
|
115
|
+
* in an uncommitted snapshot, for a workflow node it is overwritten by the next
|
|
116
|
+
* attempt — and in neither case does a watermark move, because nothing here
|
|
117
|
+
* reaches a commit.
|
|
118
|
+
*
|
|
119
|
+
* ## The one shape that would deadlock, and how it is avoided
|
|
120
|
+
*
|
|
121
|
+
* The writer runs as a floating loop and the reader is driven by the consumer
|
|
122
|
+
* pulling rows. Awaiting the writer before yielding the first row is the one
|
|
123
|
+
* arrangement that hangs: the child fills its stdout buffer with rows nobody is
|
|
124
|
+
* draining, stops reading stdin, and this side waits forever for a `drain` that
|
|
125
|
+
* requires the reader that has not started. Written the way it is, a slow
|
|
126
|
+
* consumer simply back-pressures the whole chain, which is the point.
|
|
127
|
+
*/
|
|
128
|
+
runStream(transform: Pick<CatalogTransform, 'language' | 'code' | 'mode'>, records: AsyncIterable<unknown>, options?: {
|
|
129
|
+
timeoutMs?: number;
|
|
130
|
+
context?: CatalogCodeContext;
|
|
131
|
+
}): Promise<TransformStream>;
|
|
91
132
|
private execute;
|
|
92
133
|
private spawn;
|
|
93
134
|
/** Cached, including the negative answer — probing on every run is wasteful. */
|