@dudousxd/nestjs-catalog 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.pipeline.d.ts +113 -14
- package/dist/catalog.pipeline.js +24 -8
- package/dist/client.d.ts +2 -1
- package/dist/client.js +10 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +9 -2
- package/dist/transform-runner.d.ts +7 -1
- package/dist/transform-runner.js +184 -26
- package/dist/transform-shape.d.ts +106 -0
- package/dist/transform-shape.js +419 -0
- package/package.json +1 -1
|
@@ -34,15 +34,22 @@ export declare function isConnectorKind(value: unknown): value is ConnectorKind;
|
|
|
34
34
|
* one: this used to be compared against string literals in the parser and
|
|
35
35
|
* spelled out again in a dropdown, and the two had no way to disagree loudly.
|
|
36
36
|
* The parser's chain also *ended* in JSON, so a format it did not recognise was
|
|
37
|
-
* not refused — it was read as JSON,
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
|
|
45
|
-
|
|
37
|
+
* not refused — it was read as JSON, so a spreadsheet handed to `JSON.parse`
|
|
38
|
+
* failed with a syntax error naming a byte offset rather than the format, and
|
|
39
|
+
* so did `format: "parquet"`.
|
|
40
|
+
*
|
|
41
|
+
* Two things distinguish the members, and everything downstream turns on one or
|
|
42
|
+
* the other. **Text or binary:** `xlsx` and `parquet` are binary, so everything
|
|
43
|
+
* that reads them takes bytes, and the other two are decoded first. **Whether
|
|
44
|
+
* there is a row boundary a reader can find without holding the whole
|
|
45
|
+
* payload:** `csv` and `ndjson` have one at every newline and `parquet` has one
|
|
46
|
+
* at every row group, so those three are read as a stream; `json` is a single
|
|
47
|
+
* value whose array may be nested inside an envelope that is only found by
|
|
48
|
+
* parsing down to it, and `xlsx` is a ZIP whose shared-string table generally
|
|
49
|
+
* has to be read before the sheet. Anything deciding something *per format*
|
|
50
|
+
* narrows against this list and answers {@link unreachableSourceFormat}.
|
|
51
|
+
*/
|
|
52
|
+
export declare const SOURCE_FORMATS: readonly ["csv", "ndjson", "json", "xlsx", "parquet"];
|
|
46
53
|
export type SourceFormat = (typeof SOURCE_FORMATS)[number];
|
|
47
54
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
48
55
|
export declare function isSourceFormat(value: unknown): value is SourceFormat;
|
|
@@ -439,18 +446,39 @@ export interface CatalogTransform {
|
|
|
439
446
|
description?: string;
|
|
440
447
|
language: TransformLanguage;
|
|
441
448
|
/**
|
|
442
|
-
*
|
|
443
|
-
* `context`, and returns the rows to store.
|
|
449
|
+
* A function over one batch, in either of two shapes.
|
|
444
450
|
*
|
|
445
451
|
* A batch rather than a record at a time: a transform that needs to look up,
|
|
446
452
|
* deduplicate or aggregate cannot do it one row at a time, and paying one
|
|
447
453
|
* process spawn per record would make any real load unusable.
|
|
448
454
|
*
|
|
455
|
+
* **The supported shape** is a module exporting a function that takes one
|
|
456
|
+
* object — a {@link CatalogTransformInput} — and returns the rows to store:
|
|
457
|
+
*
|
|
458
|
+
* ```js
|
|
459
|
+
* export default function transform({ records, context }) {
|
|
460
|
+
* return records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }));
|
|
461
|
+
* }
|
|
462
|
+
* ```
|
|
463
|
+
*
|
|
464
|
+
* One object rather than positional parameters, because the object is the
|
|
465
|
+
* only shape that can gain a field later. `context` arrived as a second
|
|
466
|
+
* positional parameter and got away with it; a third would have redefined
|
|
467
|
+
* what every signature already written means.
|
|
468
|
+
*
|
|
469
|
+
* **The bare-body shape** — the text between a function's braces, with
|
|
470
|
+
* `records` and `context` simply in scope — is what every transform stored
|
|
471
|
+
* before that shape existed is written in, and it keeps running byte for
|
|
472
|
+
* byte: same wrapper, same interpreter flags, same everything. See
|
|
473
|
+
* `transform-shape.ts` for how the two are told apart and why the rule cannot
|
|
474
|
+
* misread one for the other.
|
|
475
|
+
*
|
|
449
476
|
* `context` is a {@link CatalogCodeContext} — the run, the node, the counts
|
|
450
477
|
* of what fed it, and the environment variables this deployment admits.
|
|
451
|
-
*
|
|
452
|
-
*
|
|
453
|
-
*
|
|
478
|
+
*
|
|
479
|
+
* Python has neither shape and needs neither: its harness writes the `def`
|
|
480
|
+
* itself, so a Python transform is a body that never states a signature, and
|
|
481
|
+
* a new field costs one generated line rather than an edit to stored code.
|
|
454
482
|
*/
|
|
455
483
|
code: string;
|
|
456
484
|
version: number;
|
|
@@ -458,6 +486,77 @@ export interface CatalogTransform {
|
|
|
458
486
|
createdAt: string;
|
|
459
487
|
updatedAt: string;
|
|
460
488
|
}
|
|
489
|
+
/**
|
|
490
|
+
* The single argument a module-shaped transform is called with.
|
|
491
|
+
*
|
|
492
|
+
* ## Why one object
|
|
493
|
+
*
|
|
494
|
+
* So that the next thing a transform needs can be added without changing what
|
|
495
|
+
* any existing transform's signature means. Positional parameters spend that
|
|
496
|
+
* option the first time they are used: `(records, context)` fixed the list at
|
|
497
|
+
* two, and a third would silently redefine every signature ever written —
|
|
498
|
+
* including the ones in a database somewhere that nobody will re-read. A field
|
|
499
|
+
* on an object is additive by construction, and a transform that never names it
|
|
500
|
+
* is untouched by it.
|
|
501
|
+
*
|
|
502
|
+
* ## What is on it, and what is not
|
|
503
|
+
*
|
|
504
|
+
* {@link records} and {@link context}, and deliberately nothing else yet.
|
|
505
|
+
*
|
|
506
|
+
* - **No `log`.** Python's harness has one, because Python's `print` used to go
|
|
507
|
+
* nowhere; JavaScript's `console.log` — and `info`, `warn`, `error`, `debug`,
|
|
508
|
+
* `trace` — is already captured in call order. A second spelling that worked
|
|
509
|
+
* only in the new shape would split the idiom for no gain.
|
|
510
|
+
* - **No `env` shortcut.** It is already `context.env`, filtered by the same
|
|
511
|
+
* credential allow-list that governs connectors. Two paths to one value is
|
|
512
|
+
* how the two come to disagree.
|
|
513
|
+
* - **No `signal`.** The timeout is a `SIGKILL` to the whole process group;
|
|
514
|
+
* there is nothing for user code to cooperate with, and an `AbortSignal` that
|
|
515
|
+
* never fires would be a promise the runner cannot keep.
|
|
516
|
+
*
|
|
517
|
+
* The point of the object is that each of those can be reconsidered later
|
|
518
|
+
* without a migration. That is the argument, not the current field list.
|
|
519
|
+
*
|
|
520
|
+
* @typeParam TRecord - what one inbound record looks like. Editor help only:
|
|
521
|
+
* types are erased before the code runs, so a wrong one is a squiggle, never a
|
|
522
|
+
* failed run. See {@link CatalogTransformFunction}.
|
|
523
|
+
*/
|
|
524
|
+
export interface CatalogTransformInput<TRecord = Record<string, unknown>> {
|
|
525
|
+
/** The batch, exactly as the source produced it. */
|
|
526
|
+
records: TRecord[];
|
|
527
|
+
/** The run, the node, the counts, and the admitted environment variables. */
|
|
528
|
+
context: CatalogCodeContext;
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* The function a module-shaped transform exports, as `export default` or as a
|
|
532
|
+
* named export called `transform`.
|
|
533
|
+
*
|
|
534
|
+
* **This type is for the editor and for nothing else.** TypeScript transforms
|
|
535
|
+
* run through Node's own type *stripping* — the annotations are erased on the
|
|
536
|
+
* way in and never checked, by this runner or by anything else — so a transform
|
|
537
|
+
* whose types are wrong runs anyway, and produces exactly the rows its code
|
|
538
|
+
* produces. What the type buys is completion on `records` and `context` while
|
|
539
|
+
* writing, and a red underline in an editor that happens to be type-aware. What
|
|
540
|
+
* it does not buy is a single guarantee at run time; the try pane is what
|
|
541
|
+
* catches a mistake.
|
|
542
|
+
*
|
|
543
|
+
* Referencing it costs nothing at run time either, and that is a property of
|
|
544
|
+
* `import type` specifically: the stripper erases the whole statement, so
|
|
545
|
+
* nothing tries to resolve `@dudousxd/nestjs-catalog/client` inside a child
|
|
546
|
+
* process that has no `node_modules` to resolve it in. A *value* import of the
|
|
547
|
+
* same module would fail — there is no package to find from the temporary
|
|
548
|
+
* directory a transform runs in.
|
|
549
|
+
*
|
|
550
|
+
* ```ts
|
|
551
|
+
* import type { CatalogTransformFunction } from '@dudousxd/nestjs-catalog/client';
|
|
552
|
+
*
|
|
553
|
+
* const transform: CatalogTransformFunction<{ 'Mgmt Cd': string }> = ({ records }) =>
|
|
554
|
+
* records.map((r) => ({ mgmtCd: r['Mgmt Cd'] }));
|
|
555
|
+
*
|
|
556
|
+
* export default transform;
|
|
557
|
+
* ```
|
|
558
|
+
*/
|
|
559
|
+
export type CatalogTransformFunction<TRecord = Record<string, unknown>> = (input: CatalogTransformInput<TRecord>) => Array<Record<string, unknown>> | Promise<Array<Record<string, unknown>>>;
|
|
461
560
|
export interface TransformResult {
|
|
462
561
|
rows: Array<Record<string, unknown>>;
|
|
463
562
|
/**
|
package/dist/catalog.pipeline.js
CHANGED
|
@@ -105,13 +105,20 @@ function isConnectorKind(value) {
|
|
|
105
105
|
* one: this used to be compared against string literals in the parser and
|
|
106
106
|
* spelled out again in a dropdown, and the two had no way to disagree loudly.
|
|
107
107
|
* The parser's chain also *ended* in JSON, so a format it did not recognise was
|
|
108
|
-
* not refused — it was read as JSON,
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
108
|
+
* not refused — it was read as JSON, so a spreadsheet handed to `JSON.parse`
|
|
109
|
+
* failed with a syntax error naming a byte offset rather than the format, and
|
|
110
|
+
* so did `format: "parquet"`.
|
|
111
|
+
*
|
|
112
|
+
* Two things distinguish the members, and everything downstream turns on one or
|
|
113
|
+
* the other. **Text or binary:** `xlsx` and `parquet` are binary, so everything
|
|
114
|
+
* that reads them takes bytes, and the other two are decoded first. **Whether
|
|
115
|
+
* there is a row boundary a reader can find without holding the whole
|
|
116
|
+
* payload:** `csv` and `ndjson` have one at every newline and `parquet` has one
|
|
117
|
+
* at every row group, so those three are read as a stream; `json` is a single
|
|
118
|
+
* value whose array may be nested inside an envelope that is only found by
|
|
119
|
+
* parsing down to it, and `xlsx` is a ZIP whose shared-string table generally
|
|
120
|
+
* has to be read before the sheet. Anything deciding something *per format*
|
|
121
|
+
* narrows against this list and answers {@link unreachableSourceFormat}.
|
|
115
122
|
*/
|
|
116
123
|
exports.SOURCE_FORMATS = [
|
|
117
124
|
/** Delimited text with a header row. The delimiter is configurable. */
|
|
@@ -121,13 +128,22 @@ exports.SOURCE_FORMATS = [
|
|
|
121
128
|
/** A JSON document, optionally with the array nested in an envelope. */
|
|
122
129
|
'json',
|
|
123
130
|
/**
|
|
124
|
-
* A spreadsheet workbook — binary, and
|
|
131
|
+
* A spreadsheet workbook — binary, and read whole.
|
|
125
132
|
*
|
|
126
133
|
* Named for the modern extension, but the reader identifies the container
|
|
127
134
|
* from its own bytes, so the legacy `.xls` and the macro-enabled `.xlsm` are
|
|
128
135
|
* this format too rather than three names for one decision.
|
|
129
136
|
*/
|
|
130
137
|
'xlsx',
|
|
138
|
+
/**
|
|
139
|
+
* Apache Parquet — binary, and read a row group at a time.
|
|
140
|
+
*
|
|
141
|
+
* A row group is a chunk boundary the format supplies rather than one a
|
|
142
|
+
* reader has to invent, which is what separates it from `xlsx`. It also
|
|
143
|
+
* carries a real type system, so unlike the text formats a value arrives as
|
|
144
|
+
* the type the writer meant rather than as a string.
|
|
145
|
+
*/
|
|
146
|
+
'parquet',
|
|
131
147
|
];
|
|
132
148
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
133
149
|
function isSourceFormat(value) {
|
package/dist/client.d.ts
CHANGED
|
@@ -153,7 +153,8 @@ export declare const catalogRoutes: {
|
|
|
153
153
|
readonly traces: () => string;
|
|
154
154
|
readonly trace: (id: string) => string;
|
|
155
155
|
};
|
|
156
|
-
export type { CatalogConnection, CatalogConnector, ConnectionCheck, CatalogTransform, CatalogWorkflow, CatalogWorkflowCapabilities, CatalogWorkflowRelease, ConnectorKind, ConnectorRun, TransformLanguage, TransformResult, CallableWorkflowRef, WorkflowBranchLabel, WorkflowCallEnvelope, WorkflowCallMode, WorkflowCallNode, WorkflowCallOutput, WorkflowEdge, WorkflowExecutionMode, WorkflowFilterAll, WorkflowFilterAny, WorkflowFilterComparison, WorkflowFilterGroup, WorkflowFilterNode, WorkflowFilterOneOf, WorkflowFilterOperator, WorkflowFilterPredicate, WorkflowFilterPredicateKind, WorkflowFilterPresence, WorkflowFilterValue, WorkflowGraph, WorkflowEnvPredicate, WorkflowIfNode, WorkflowIfPredicate, WorkflowIssueCode, WorkflowNode, WorkflowNodeKind, WorkflowNodeOutcome, WorkflowPredicateKind, WorkflowRowCountPredicate, WorkflowRunOrderEntry, WorkflowSinkNode, WorkflowSkipReason, WorkflowSourceNode, WorkflowStageRef, WorkflowTransformNode, WorkflowValidationIssue, CatalogReusableNode, CatalogReusableNodeUse, ReusableNodeBody, ReusableNodeKind, ReusableNodeRef, ReusableSinkBody, ReusableSourceBody, SourceFormat, VersionPinCopy, } from './catalog.pipeline';
|
|
156
|
+
export type { CatalogConnection, CatalogConnector, ConnectionCheck, CatalogTransform, CatalogTransformFunction, CatalogTransformInput, CatalogWorkflow, CatalogWorkflowCapabilities, CatalogWorkflowRelease, ConnectorKind, ConnectorRun, TransformLanguage, TransformResult, CallableWorkflowRef, WorkflowBranchLabel, WorkflowCallEnvelope, WorkflowCallMode, WorkflowCallNode, WorkflowCallOutput, WorkflowEdge, WorkflowExecutionMode, WorkflowFilterAll, WorkflowFilterAny, WorkflowFilterComparison, WorkflowFilterGroup, WorkflowFilterNode, WorkflowFilterOneOf, WorkflowFilterOperator, WorkflowFilterPredicate, WorkflowFilterPredicateKind, WorkflowFilterPresence, WorkflowFilterValue, WorkflowGraph, WorkflowEnvPredicate, WorkflowIfNode, WorkflowIfPredicate, WorkflowIssueCode, WorkflowNode, WorkflowNodeKind, WorkflowNodeOutcome, WorkflowPredicateKind, WorkflowRowCountPredicate, WorkflowRunOrderEntry, WorkflowSinkNode, WorkflowSkipReason, WorkflowSourceNode, WorkflowStageRef, WorkflowTransformNode, WorkflowValidationIssue, CatalogReusableNode, CatalogReusableNodeUse, ReusableNodeBody, ReusableNodeKind, ReusableNodeRef, ReusableSinkBody, ReusableSourceBody, SourceFormat, VersionPinCopy, } from './catalog.pipeline';
|
|
157
|
+
export { type TransformShape, transformDeclaresModule, transformShape, } from './transform-shape';
|
|
157
158
|
export { CONNECTOR_KINDS, isConnectorKind, isSourceFormat, isTransformLanguage, SOURCE_FORMATS, isWorkflowEdge, isWorkflowNode, liveWorkflowVersion, REDACTED_SECRET, TRANSFORM_LANGUAGES, readWorkflowCallOutput, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, isWorkflowCallMode, unreachableCallMode, workflowCallMode, applyReusableNode, reusableNodeBodyOf, isReusableNodeBody, isReusableNodeKind, REUSABLE_NODE_KINDS, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, unreachableReusableNodeKind, describeLiveVersion, describeVersionPin, } from './catalog.pipeline';
|
|
158
159
|
/**
|
|
159
160
|
* The workflow validator, shipped to the browser deliberately.
|
package/dist/client.js
CHANGED
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* types are.
|
|
12
12
|
*/
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
-
exports.
|
|
15
|
-
exports.DELETE_RECONCILIATION_STRATEGIES = exports.callableWorkflowBlock = exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableFilterPredicateKind = exports.unreachableFilterOperator = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowFilterMatches = exports.isWorkflowSkipReason = exports.isWorkflowPredicateKind = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowBranchLabel = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = void 0;
|
|
14
|
+
exports.workflowColumnX = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_COLUMN_GAP = exports.describeVersionPin = exports.describeLiveVersion = exports.unreachableReusableNodeKind = exports.nodeKindIsReusable = exports.NODE_KIND_IS_REUSABLE = exports.REUSABLE_NODE_KINDS = exports.isReusableNodeKind = exports.isReusableNodeBody = exports.reusableNodeBodyOf = exports.applyReusableNode = exports.workflowCallMode = exports.unreachableCallMode = exports.isWorkflowCallMode = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_CALL_CONTRACT = exports.readWorkflowCallOutput = exports.TRANSFORM_LANGUAGES = exports.REDACTED_SECRET = exports.liveWorkflowVersion = exports.isWorkflowNode = exports.isWorkflowEdge = exports.SOURCE_FORMATS = exports.isTransformLanguage = exports.isSourceFormat = exports.isConnectorKind = exports.CONNECTOR_KINDS = exports.transformShape = exports.transformDeclaresModule = exports.catalogRoutes = exports.UnsafeIdentifierError = exports.physicalColumn = exports.outputAlias = exports.isSafeIdentifier = exports.VALUELESS_FILTER_OPERATORS = exports.resolveObjectFilters = exports.parseObjectFilter = exports.offeredFilterOperators = exports.isCatalogFilterOperator = exports.filterOperatorsFor = exports.filterOperatorTakesValue = exports.encodeObjectFilter = exports.coerceFilterValue = exports.CATALOG_FILTER_OPERATORS = exports.CATALOG_FILTER_LIMIT = exports.CATALOG_REVISION_LIMIT = void 0;
|
|
15
|
+
exports.DELETE_RECONCILIATION_STRATEGIES = exports.callableWorkflowBlock = exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableFilterPredicateKind = exports.unreachableFilterOperator = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowFilterMatches = exports.isWorkflowSkipReason = exports.isWorkflowPredicateKind = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowBranchLabel = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.workflowRowY = void 0;
|
|
16
16
|
exports.isDeleteReconciliationStrategy = isDeleteReconciliationStrategy;
|
|
17
17
|
exports.pipelineExpectationRoutes = pipelineExpectationRoutes;
|
|
18
18
|
// A value, not a type: a screen saying how far back the history goes should read
|
|
@@ -129,6 +129,14 @@ exports.catalogRoutes = {
|
|
|
129
129
|
traces: () => '/catalog/events/traces',
|
|
130
130
|
trace: (id) => `/catalog/events/traces/${encodeURIComponent(id)}`,
|
|
131
131
|
};
|
|
132
|
+
// The transform shape rule, as the runner itself applies it. A browser build
|
|
133
|
+
// can take this — `transform-shape.ts` imports nothing at all — and an editor
|
|
134
|
+
// that tells the author which shape their code is in should be reading the
|
|
135
|
+
// answer rather than reproducing the reasoning. A second copy would be the one
|
|
136
|
+
// that disagrees on the day it matters.
|
|
137
|
+
var transform_shape_1 = require("./transform-shape");
|
|
138
|
+
Object.defineProperty(exports, "transformDeclaresModule", { enumerable: true, get: function () { return transform_shape_1.transformDeclaresModule; } });
|
|
139
|
+
Object.defineProperty(exports, "transformShape", { enumerable: true, get: function () { return transform_shape_1.transformShape; } });
|
|
132
140
|
// Values, not types: a form that offers the kinds should read them from here
|
|
133
141
|
// rather than keeping a copy that drifts.
|
|
134
142
|
var catalog_pipeline_1 = require("./catalog.pipeline");
|
package/dist/index.d.ts
CHANGED
|
@@ -8,12 +8,13 @@ export { type CatalogOverlayStore, FileCatalogOverlayStore, InMemoryCatalogOverl
|
|
|
8
8
|
export { CATALOG_OVERLAY_STORE } from './catalog.overlay-store.token';
|
|
9
9
|
export { MikroOrmCatalogRegistry } from './catalog.registry';
|
|
10
10
|
export { CatalogRegistry } from './catalog.registry.base';
|
|
11
|
-
export { CATALOG_PIPELINE_STORE, CODE_CONTEXT_CONTRACT, type CatalogCodeContext, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogLoadExpectations, type CatalogLoadExpectationStore, type CatalogPipelineStore, type CatalogStageStore, type CatalogTransform, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowRelease, type CatalogWorkflowReleaseStore, type CatalogWorkflowStore, type CallableWorkflowBlock, type CallableWorkflowDisagreement, type CallableWorkflowRef, callableWorkflowBlock, type ConnectorKind, type ConnectorRun, type DeleteReconciliation, isConnectorKind, isPipelineStore, isSourceFormat, isTransformLanguage, type LoadExpectation, type RowCountBound, SOURCE_FORMATS, type SourceFormat, unreachableSourceFormat, type StoredLoadExpectation, readWorkflowCallOutput, REDACTED_SECRET, supportsLoadExpectations, supportsReusableNodes, supportsTransformPins, supportsTransformRevisions, applyReusableNode, type CatalogReusableNode, type CatalogReusableNodeStore, type CatalogReusableNodeUse, describeLiveVersion, describeVersionPin, isReusableNodeBody, isReusableNodeKind, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, REUSABLE_NODE_KINDS, type ReusableNodeBody, type ReusableNodeKind, type ReusableNodeRef, type ReusableSinkBody, type ReusableSourceBody, reusableNodeBodyOf, unreachableReusableNodeKind, type VersionPinCopy, isWorkflowBranchLabel, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowNode, isWorkflowCallMode, isWorkflowNodeKind, isWorkflowPredicateKind, isWorkflowSkipReason, isWorkflowStatus, liveWorkflowVersion, supportsWorkflowReleases, supportsWorkflows, supportsWorkflowStages, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, type TransformLanguage, type TransformResult, type TransformRunner, unreachableFilterOperator, unreachableFilterPredicateKind, unreachableCallMode, unreachableNodeKind, unreachablePredicateKind, validateWorkflow, WORKFLOW_BRANCH_LABELS, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, WORKFLOW_COLUMN_GAP, WORKFLOW_EXECUTION_MODES, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_HEIGHT, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_NODE_WIDTH, WORKFLOW_PREDICATE_KINDS, WORKFLOW_ROW_GAP, WORKFLOW_SKIP_REASONS, WORKFLOW_STATUSES, type WorkflowBranchLabel, workflowColumnX, workflowRowY, type WorkflowCallEnvelope, type WorkflowCallMode, workflowCallMode, type WorkflowCallNode, type WorkflowCallOutput, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowFilterAll, type WorkflowFilterAny, type WorkflowFilterComparison, type WorkflowFilterGroup, workflowFilterMatches, type WorkflowFilterNode, type WorkflowFilterOneOf, type WorkflowFilterOperator, type WorkflowFilterPredicate, type WorkflowFilterPredicateKind, type WorkflowFilterPresence, type WorkflowFilterValue, type WorkflowGraph, workflowGraphHash, type WorkflowEnvPredicate, type WorkflowIfNode, type WorkflowIfPredicate, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, workflowNarrowedTypes, type WorkflowNodeOutcome, workflowNodeRuns, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowRunOrderEntry, type WorkflowPredicateKind, type WorkflowRowCountPredicate, type WorkflowSinkNode, type WorkflowSkipReason, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowStatus, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
|
|
11
|
+
export { CATALOG_PIPELINE_STORE, CODE_CONTEXT_CONTRACT, type CatalogCodeContext, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogLoadExpectations, type CatalogLoadExpectationStore, type CatalogPipelineStore, type CatalogStageStore, type CatalogTransform, type CatalogTransformFunction, type CatalogTransformInput, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowRelease, type CatalogWorkflowReleaseStore, type CatalogWorkflowStore, type CallableWorkflowBlock, type CallableWorkflowDisagreement, type CallableWorkflowRef, callableWorkflowBlock, type ConnectorKind, type ConnectorRun, type DeleteReconciliation, isConnectorKind, isPipelineStore, isSourceFormat, isTransformLanguage, type LoadExpectation, type RowCountBound, SOURCE_FORMATS, type SourceFormat, unreachableSourceFormat, type StoredLoadExpectation, readWorkflowCallOutput, REDACTED_SECRET, supportsLoadExpectations, supportsReusableNodes, supportsTransformPins, supportsTransformRevisions, applyReusableNode, type CatalogReusableNode, type CatalogReusableNodeStore, type CatalogReusableNodeUse, describeLiveVersion, describeVersionPin, isReusableNodeBody, isReusableNodeKind, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, REUSABLE_NODE_KINDS, type ReusableNodeBody, type ReusableNodeKind, type ReusableNodeRef, type ReusableSinkBody, type ReusableSourceBody, reusableNodeBodyOf, unreachableReusableNodeKind, type VersionPinCopy, isWorkflowBranchLabel, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowNode, isWorkflowCallMode, isWorkflowNodeKind, isWorkflowPredicateKind, isWorkflowSkipReason, isWorkflowStatus, liveWorkflowVersion, supportsWorkflowReleases, supportsWorkflows, supportsWorkflowStages, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, type TransformLanguage, type TransformResult, type TransformRunner, unreachableFilterOperator, unreachableFilterPredicateKind, unreachableCallMode, unreachableNodeKind, unreachablePredicateKind, validateWorkflow, WORKFLOW_BRANCH_LABELS, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, WORKFLOW_COLUMN_GAP, WORKFLOW_EXECUTION_MODES, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_HEIGHT, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_NODE_WIDTH, WORKFLOW_PREDICATE_KINDS, WORKFLOW_ROW_GAP, WORKFLOW_SKIP_REASONS, WORKFLOW_STATUSES, type WorkflowBranchLabel, workflowColumnX, workflowRowY, type WorkflowCallEnvelope, type WorkflowCallMode, workflowCallMode, type WorkflowCallNode, type WorkflowCallOutput, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowFilterAll, type WorkflowFilterAny, type WorkflowFilterComparison, type WorkflowFilterGroup, workflowFilterMatches, type WorkflowFilterNode, type WorkflowFilterOneOf, type WorkflowFilterOperator, type WorkflowFilterPredicate, type WorkflowFilterPredicateKind, type WorkflowFilterPresence, type WorkflowFilterValue, type WorkflowGraph, workflowGraphHash, type WorkflowEnvPredicate, type WorkflowIfNode, type WorkflowIfPredicate, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, workflowNarrowedTypes, type WorkflowNodeOutcome, workflowNodeRuns, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowRunOrderEntry, type WorkflowPredicateKind, type WorkflowRowCountPredicate, type WorkflowSinkNode, type WorkflowSkipReason, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowStatus, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
|
|
12
12
|
export { type ColumnarStageBatch, STAGE_ENCODING, STAGE_ENCODING_VERSION, type StagePayload, classifyStagePayload, decodeStageRows, encodeStageRows, isColumnarStageBatch, } from './catalog.stage-encoding';
|
|
13
13
|
export * from './catalog.environment';
|
|
14
14
|
export { QueryCache } from './catalog.query-cache';
|
|
15
15
|
export { type CsvRow, csvCell, csvLines, guardFormula, toCsv } from './catalog.csv';
|
|
16
16
|
export { SubprocessTransformRunner, type TransformRunnerOptions, } from './transform-runner';
|
|
17
|
+
export { type TransformShape, transformDeclaresModule, transformShape, transformShapeHint, } from './transform-shape';
|
|
17
18
|
export { CatalogService } from './catalog.service';
|
|
18
19
|
export { DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT, bestMatch, emptySearch, maySearch, type SearchInput, type SearchableDashboard, type SearchableSavedQuery, searchCatalog, visibleToPrincipal, } from './search';
|
|
19
20
|
export type { CatalogSearchField, CatalogSearchHit, CatalogSearchKind, CatalogSearchRank, CatalogSearchResult, } from './search.types';
|
package/dist/index.js
CHANGED
|
@@ -16,8 +16,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
exports.isWorkflowExecutionMode = exports.isWorkflowEdge = exports.isWorkflowBranchLabel = exports.unreachableReusableNodeKind = exports.reusableNodeBodyOf = exports.REUSABLE_NODE_KINDS = exports.nodeKindIsReusable = exports.NODE_KIND_IS_REUSABLE = exports.isReusableNodeKind = exports.isReusableNodeBody = exports.describeVersionPin = exports.describeLiveVersion = exports.applyReusableNode = exports.supportsTransformRevisions = exports.supportsTransformPins = exports.supportsReusableNodes = exports.supportsLoadExpectations = exports.REDACTED_SECRET = exports.readWorkflowCallOutput = exports.unreachableSourceFormat = exports.SOURCE_FORMATS = exports.isTransformLanguage = exports.isSourceFormat = exports.isPipelineStore = exports.isConnectorKind = exports.callableWorkflowBlock = exports.CONNECTOR_KINDS = exports.CODE_CONTEXT_CONTRACT = exports.CATALOG_PIPELINE_STORE = exports.CatalogRegistry = exports.MikroOrmCatalogRegistry = exports.CATALOG_OVERLAY_STORE = exports.InMemoryCatalogOverlayStore = exports.FileCatalogOverlayStore = exports.CATALOG_OPTIONS = exports.isStreamingQueryStore = exports.isQueryStore = exports.assertReadOnlyShape = exports.CatalogModule = exports.emitCatalog = exports.curationActor = exports.channelNameFor = exports.catalogEventPhase = exports.UNATTRIBUTED_PRINCIPAL_ID = exports.CATALOG_LIB = exports.CATALOG_EVENTS = exports.CATALOG_EVENT_PHASE_FALLBACK = exports.CATALOG_EVENT_PHASE = exports.CatalogType = exports.CatalogProperty = void 0;
|
|
18
18
|
exports.workflowRunOrder = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowGraphHash = exports.workflowFilterMatches = exports.workflowCallMode = exports.workflowRowY = exports.workflowColumnX = exports.WORKFLOW_STATUSES = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_BRANCH_LABELS = exports.validateWorkflow = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableCallMode = exports.unreachableFilterPredicateKind = exports.unreachableFilterOperator = exports.TRANSFORM_LANGUAGES = exports.TRANSFORM_RUNNER = exports.supportsWorkflowStages = exports.supportsWorkflows = exports.supportsWorkflowReleases = exports.liveWorkflowVersion = exports.isWorkflowStatus = exports.isWorkflowSkipReason = exports.isWorkflowPredicateKind = exports.isWorkflowNodeKind = exports.isWorkflowCallMode = exports.isWorkflowNode = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = void 0;
|
|
19
|
-
exports.
|
|
20
|
-
exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.UnsafeIdentifierError = exports.supportsCarryForward = exports.physicalColumn = exports.outputAlias = exports.isWriteStore = exports.isSafeIdentifier = exports.isReservedColumn = exports.isCatalogStoreCapabilities = void 0;
|
|
19
|
+
exports.CATALOG_STORE = exports.CATALOG_SNAPSHOT_MODES = exports.CATALOG_RESERVED_COLUMNS = exports.assertSafeIdentifier = exports.assertNoColumnCollisions = exports.StaticKeyPrincipalResolver = exports.readableObjectPage = exports.mayWrite = exports.mayRead = exports.maySeeClassification = exports.PRINCIPAL_ACTOR_SEPARATOR = exports.parsePrincipalId = exports.hasScope = exports.expandScopes = exports.delegatePrincipal = exports.composePrincipalId = exports.CATALOG_PRINCIPAL_RESOLVER = exports.traceOutcomeFilter = exports.supportsSavedQueryRevisions = exports.isWorkspaceStore = exports.isTraceStore = exports.isCatalogTraceOutcome = exports.embeddedVisualization = exports.CATALOG_WORKSPACE_STORE = exports.CATALOG_TRACE_STORE = exports.CATALOG_TRACE_OUTCOMES = exports.CATALOG_REVISION_LIMIT = exports.visibleToPrincipal = exports.searchCatalog = exports.maySearch = exports.emptySearch = exports.bestMatch = exports.MAX_SEARCH_LIMIT = exports.DEFAULT_SEARCH_LIMIT = exports.CatalogService = exports.transformShapeHint = exports.transformShape = exports.transformDeclaresModule = exports.SubprocessTransformRunner = exports.toCsv = exports.guardFormula = exports.csvLines = exports.csvCell = exports.QueryCache = exports.isColumnarStageBatch = exports.encodeStageRows = exports.decodeStageRows = exports.classifyStagePayload = exports.STAGE_ENCODING_VERSION = exports.STAGE_ENCODING = void 0;
|
|
20
|
+
exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.UnsafeIdentifierError = exports.supportsCarryForward = exports.physicalColumn = exports.outputAlias = exports.isWriteStore = exports.isSafeIdentifier = exports.isReservedColumn = exports.isCatalogStoreCapabilities = exports.findColumnCollisions = exports.supportsObjectFilters = exports.CatalogColumnCollisionError = void 0;
|
|
21
21
|
var catalog_decorators_1 = require("./catalog.decorators");
|
|
22
22
|
Object.defineProperty(exports, "CatalogProperty", { enumerable: true, get: function () { return catalog_decorators_1.CatalogProperty; } });
|
|
23
23
|
Object.defineProperty(exports, "CatalogType", { enumerable: true, get: function () { return catalog_decorators_1.CatalogType; } });
|
|
@@ -164,6 +164,13 @@ Object.defineProperty(exports, "guardFormula", { enumerable: true, get: function
|
|
|
164
164
|
Object.defineProperty(exports, "toCsv", { enumerable: true, get: function () { return catalog_csv_1.toCsv; } });
|
|
165
165
|
var transform_runner_1 = require("./transform-runner");
|
|
166
166
|
Object.defineProperty(exports, "SubprocessTransformRunner", { enumerable: true, get: function () { return transform_runner_1.SubprocessTransformRunner; } });
|
|
167
|
+
// One rule for which shape a transform's code is in, exported so the editor can
|
|
168
|
+
// show the author what the runner concluded instead of maintaining a second
|
|
169
|
+
// copy that says something else. Same reason `isTransformLanguage` is exported.
|
|
170
|
+
var transform_shape_1 = require("./transform-shape");
|
|
171
|
+
Object.defineProperty(exports, "transformDeclaresModule", { enumerable: true, get: function () { return transform_shape_1.transformDeclaresModule; } });
|
|
172
|
+
Object.defineProperty(exports, "transformShape", { enumerable: true, get: function () { return transform_shape_1.transformShape; } });
|
|
173
|
+
Object.defineProperty(exports, "transformShapeHint", { enumerable: true, get: function () { return transform_shape_1.transformShapeHint; } });
|
|
167
174
|
var catalog_service_1 = require("./catalog.service");
|
|
168
175
|
Object.defineProperty(exports, "CatalogService", { enumerable: true, get: function () { return catalog_service_1.CatalogService; } });
|
|
169
176
|
// Search. The result types are on `/client` too, for a browser; these are here
|
|
@@ -28,7 +28,12 @@ export interface TransformRunnerOptions {
|
|
|
28
28
|
* - it runs in a working directory of this runner's choosing but on the host's
|
|
29
29
|
* filesystem, so a service account token under
|
|
30
30
|
* `/var/run/secrets/kubernetes.io/serviceaccount/` is an absolute path away;
|
|
31
|
-
* - it can open sockets, as whatever user the service runs as
|
|
31
|
+
* - it can open sockets, as whatever user the service runs as;
|
|
32
|
+
* - a module-shaped transform is written to a file in that temporary directory
|
|
33
|
+
* for the length of the run, so its own source is briefly on disk. That is
|
|
34
|
+
* not a new exposure — the code is the thing running, and it can read itself
|
|
35
|
+
* from anywhere — but it is a fact worth having written down next to the
|
|
36
|
+
* others rather than discovered in a directory listing.
|
|
32
37
|
*
|
|
33
38
|
* So the allowlist is a guard rail against the accident, and the reachability of
|
|
34
39
|
* everything it names is a property of the process boundary, not a leak to be
|
|
@@ -83,6 +88,7 @@ export declare class SubprocessTransformRunner implements TransformRunner {
|
|
|
83
88
|
timeoutMs?: number;
|
|
84
89
|
context?: CatalogCodeContext;
|
|
85
90
|
}): Promise<TransformResult>;
|
|
91
|
+
private execute;
|
|
86
92
|
private spawn;
|
|
87
93
|
/** Cached, including the negative answer — probing on every run is wasteful. */
|
|
88
94
|
private resolvePython;
|
package/dist/transform-runner.js
CHANGED
|
@@ -12,11 +12,15 @@ var SubprocessTransformRunner_1;
|
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.SubprocessTransformRunner = void 0;
|
|
14
14
|
const node_child_process_1 = require("node:child_process");
|
|
15
|
+
const node_crypto_1 = require("node:crypto");
|
|
15
16
|
const node_fs_1 = require("node:fs");
|
|
17
|
+
const promises_1 = require("node:fs/promises");
|
|
16
18
|
const node_os_1 = require("node:os");
|
|
17
19
|
const node_path_1 = require("node:path");
|
|
20
|
+
const node_url_1 = require("node:url");
|
|
18
21
|
const common_1 = require("@nestjs/common");
|
|
19
22
|
const catalog_pipeline_1 = require("./catalog.pipeline");
|
|
23
|
+
const transform_shape_1 = require("./transform-shape");
|
|
20
24
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
21
25
|
const MAX_OUTPUT_BYTES = 32 * 1024 * 1024;
|
|
22
26
|
/**
|
|
@@ -127,7 +131,12 @@ const REPORTED_PACKAGES = ['pandas', 'numpy', 'pyarrow', 'requests'];
|
|
|
127
131
|
* - it runs in a working directory of this runner's choosing but on the host's
|
|
128
132
|
* filesystem, so a service account token under
|
|
129
133
|
* `/var/run/secrets/kubernetes.io/serviceaccount/` is an absolute path away;
|
|
130
|
-
* - it can open sockets, as whatever user the service runs as
|
|
134
|
+
* - it can open sockets, as whatever user the service runs as;
|
|
135
|
+
* - a module-shaped transform is written to a file in that temporary directory
|
|
136
|
+
* for the length of the run, so its own source is briefly on disk. That is
|
|
137
|
+
* not a new exposure — the code is the thing running, and it can read itself
|
|
138
|
+
* from anywhere — but it is a fact worth having written down next to the
|
|
139
|
+
* others rather than discovered in a directory listing.
|
|
131
140
|
*
|
|
132
141
|
* So the allowlist is a guard rail against the accident, and the reachability of
|
|
133
142
|
* everything it names is a property of the process boundary, not a leak to be
|
|
@@ -223,24 +232,40 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
|
|
|
223
232
|
if (!interpreter) {
|
|
224
233
|
throw new Error('No python3 on PATH, so python transforms cannot run here. Use javascript or typescript, or install python in the image.');
|
|
225
234
|
}
|
|
226
|
-
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
235
|
+
// Python is not asked: its harness writes the `def`, so a Python transform
|
|
236
|
+
// never states a signature and has nothing to detect. See
|
|
237
|
+
// {@link pythonHarness}.
|
|
238
|
+
const shape = python ? 'body' : (0, transform_shape_1.transformShape)(transform.code);
|
|
239
|
+
// Written to disk only for the module shape, and only for the length of the
|
|
240
|
+
// run. A module has to be *imported* to be a module — its `export default`
|
|
241
|
+
// creates no binding this harness could name, and rewriting the keyword
|
|
242
|
+
// into an assignment would be surgery on somebody's source. A file also
|
|
243
|
+
// gives Node the extension it needs to strip TypeScript, and gives the
|
|
244
|
+
// author stack frames with real line numbers instead of `[eval]`.
|
|
245
|
+
const modulePath = shape === 'module'
|
|
246
|
+
? (0, node_path_1.join)((0, node_os_1.tmpdir)(), `catalog-transform-${(0, node_crypto_1.randomUUID)()}.${transform.language === 'typescript' ? 'mts' : 'mjs'}`)
|
|
247
|
+
: undefined;
|
|
248
|
+
try {
|
|
249
|
+
if (modulePath)
|
|
250
|
+
await (0, promises_1.writeFile)(modulePath, transform.code, 'utf8');
|
|
251
|
+
const args = interpreterArgs(transform, modulePath);
|
|
252
|
+
return await this.execute(interpreter, args, records, context, timeoutMs, shape, started);
|
|
253
|
+
}
|
|
254
|
+
finally {
|
|
255
|
+
// Unlinked whether the run returned, threw, or was killed on the timeout —
|
|
256
|
+
// the parent settles in all three, so nothing is left in `tmpdir` for an
|
|
257
|
+
// operator to find later and wonder about.
|
|
258
|
+
if (modulePath)
|
|
259
|
+
await (0, promises_1.rm)(modulePath, { force: true });
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
async execute(interpreter, args, records, context, timeoutMs, shape, started) {
|
|
238
263
|
// An envelope rather than the bare array stdin used to carry. The context
|
|
239
264
|
// travels beside the records rather than in the child's `env`, and that is
|
|
240
265
|
// the deliberate half of it: the child's own environment stays
|
|
241
266
|
// `{PATH, NODE_ENV}`, so nothing about what a transform may read changes by
|
|
242
267
|
// accident when somebody edits the spawn options later.
|
|
243
|
-
const { stdout, stderr } = await this.spawn(interpreter, args, JSON.stringify({ records, context }), timeoutMs);
|
|
268
|
+
const { stdout, stderr } = await this.spawn(interpreter, args, JSON.stringify({ records, context }), timeoutMs, shape);
|
|
244
269
|
let parsed;
|
|
245
270
|
try {
|
|
246
271
|
// The harness prints exactly one JSON line last; anything the code wrote
|
|
@@ -262,7 +287,7 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
|
|
|
262
287
|
elapsedMs: Date.now() - started,
|
|
263
288
|
};
|
|
264
289
|
}
|
|
265
|
-
spawn(command, args, input, timeoutMs) {
|
|
290
|
+
spawn(command, args, input, timeoutMs, shape = 'body') {
|
|
266
291
|
return new Promise((resolve, reject) => {
|
|
267
292
|
const child = (0, node_child_process_1.spawn)(command, args, {
|
|
268
293
|
// An empty environment, not the parent's. A transform has no business
|
|
@@ -321,7 +346,11 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
|
|
|
321
346
|
settled = true;
|
|
322
347
|
clearTimeout(timer);
|
|
323
348
|
if (code !== 0 && stdout.trim().length === 0) {
|
|
324
|
-
|
|
349
|
+
// The shape hint rides along here specifically: a body whose author
|
|
350
|
+
// meant it as a module fails before the harness's own try/catch is
|
|
351
|
+
// ever entered, so this branch is the only place that error can be
|
|
352
|
+
// annotated. See {@link transformShapeHint}.
|
|
353
|
+
reject(new Error(`The transform exited with code ${code}. ${stderr.slice(0, 500)}${(0, transform_shape_1.transformShapeHint)(shape, stderr)}`));
|
|
325
354
|
return;
|
|
326
355
|
}
|
|
327
356
|
resolve({ stdout, stderr });
|
|
@@ -369,6 +398,27 @@ exports.SubprocessTransformRunner = SubprocessTransformRunner = SubprocessTransf
|
|
|
369
398
|
(0, common_1.Injectable)(),
|
|
370
399
|
__metadata("design:paramtypes", [Object])
|
|
371
400
|
], SubprocessTransformRunner);
|
|
401
|
+
/**
|
|
402
|
+
* What the interpreter is invoked with, and which harness it is handed.
|
|
403
|
+
*
|
|
404
|
+
* `module-typescript` is Node's own stripping — types are erased, never
|
|
405
|
+
* checked. A transform with a wrong type still runs; the editor's try pane is
|
|
406
|
+
* what catches it, not the compiler.
|
|
407
|
+
*
|
|
408
|
+
* The module shape needs none of that flag here: the harness itself is plain
|
|
409
|
+
* JavaScript, and the author's `.mts` file is stripped on import by its
|
|
410
|
+
* extension. That confines the stripper to the code that asked for it, rather
|
|
411
|
+
* than running this file's own generated source through it as well.
|
|
412
|
+
*/
|
|
413
|
+
function interpreterArgs(transform, modulePath) {
|
|
414
|
+
if (transform.language === 'python')
|
|
415
|
+
return ['-c', pythonHarness(transform.code)];
|
|
416
|
+
const script = modulePath
|
|
417
|
+
? javascriptModuleHarness((0, node_url_1.pathToFileURL)(modulePath).href)
|
|
418
|
+
: javascriptHarness(transform.code);
|
|
419
|
+
const inputType = transform.language === 'typescript' && !modulePath ? 'module-typescript' : 'module';
|
|
420
|
+
return ['--input-type', inputType, '-e', script];
|
|
421
|
+
}
|
|
372
422
|
/**
|
|
373
423
|
* The context for a run that has none: a spec, or a host driving the runner by
|
|
374
424
|
* hand.
|
|
@@ -437,7 +487,15 @@ function withFinalLogs(error, logs) {
|
|
|
437
487
|
return `${error}\n${heading}\n${tail.map((line) => ` ${line}`).join('\n')}`;
|
|
438
488
|
}
|
|
439
489
|
/**
|
|
440
|
-
* The JavaScript and TypeScript harness
|
|
490
|
+
* The JavaScript and TypeScript harness for the **bare-body** shape: the
|
|
491
|
+
* author's code is the inside of a function this string writes.
|
|
492
|
+
*
|
|
493
|
+
* Unchanged, and that is the feature. Every transform stored before the module
|
|
494
|
+
* shape existed is a bare body, and it runs through the identical wrapper with
|
|
495
|
+
* the identical positional parameters and the identical interpreter flags — the
|
|
496
|
+
* new shape is a second path beside this one, not a rewrite of it. See
|
|
497
|
+
* `transform-shape.ts` for the rule that decides which path a given piece of
|
|
498
|
+
* code takes, and why a stored transform cannot be sent down the wrong one.
|
|
441
499
|
*
|
|
442
500
|
* `console.log` is captured rather than left on stdout so user code cannot
|
|
443
501
|
* corrupt the single JSON line this prints — a transform that logs a `{` would
|
|
@@ -462,7 +520,84 @@ function withFinalLogs(error, logs) {
|
|
|
462
520
|
* gets as far as being serialised.
|
|
463
521
|
*/
|
|
464
522
|
function javascriptHarness(code) {
|
|
465
|
-
return
|
|
523
|
+
return `${JAVASCRIPT_PRELUDE}
|
|
524
|
+
try {
|
|
525
|
+
${JAVASCRIPT_PAYLOAD}
|
|
526
|
+
const transform = async (records, context) => { ${code} };
|
|
527
|
+
const rows = await transform(records, context);
|
|
528
|
+
process.stdout.write(JSON.stringify({ rows: rows ?? [], logs: captured() }));
|
|
529
|
+
} catch (error) {
|
|
530
|
+
${JAVASCRIPT_FAILURE}
|
|
531
|
+
}
|
|
532
|
+
`;
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* The harness for the module shape: import the author's module, call what it
|
|
536
|
+
* exports with one object.
|
|
537
|
+
*
|
|
538
|
+
* Everything above the call is the same prelude the bare-body harness uses —
|
|
539
|
+
* the same six console channels, the same two caps, the same envelope, the same
|
|
540
|
+
* frozen `context`. A transform's log behaviour changing because of the shape it
|
|
541
|
+
* happens to be written in would be exactly as surprising as it changing because
|
|
542
|
+
* of the language, and the constants say why that is not allowed to happen.
|
|
543
|
+
*
|
|
544
|
+
* The code arrives as a **file URL**, not as text spliced into this string, and
|
|
545
|
+
* the difference matters three times over. `export default` binds nothing that
|
|
546
|
+
* an enclosing scope could name, so the module genuinely has to be imported;
|
|
547
|
+
* `.mts` is what tells Node to strip the types, so the extension does the job
|
|
548
|
+
* `--input-type module-typescript` does for a body; and a stack frame reads
|
|
549
|
+
* `catalog-transform-….mts:3:11` rather than `[eval]`, which is the difference
|
|
550
|
+
* between a line number and a shrug.
|
|
551
|
+
*
|
|
552
|
+
* ## What it accepts, and what it refuses
|
|
553
|
+
*
|
|
554
|
+
* `export default`, or a named export called `transform`. Two spellings rather
|
|
555
|
+
* than one because both are things people write without being told to, and
|
|
556
|
+
* because both are *real exports* — neither is a guess about a name in scope.
|
|
557
|
+
*
|
|
558
|
+
* A module that exports neither is **refused, by name**. The alternative is a
|
|
559
|
+
* transform that returns no rows and says nothing about why, which downstream
|
|
560
|
+
* reads as a source that produced nothing — a connector would commit an empty
|
|
561
|
+
* snapshot over live data on the strength of a missing `default` keyword.
|
|
562
|
+
*
|
|
563
|
+
* The import is deliberately not wrapped in a fallback to the body shape. Code
|
|
564
|
+
* that fails to parse as a module has one honest answer — the parse error, with
|
|
565
|
+
* the line — and re-running it in the other shape would replace that with a
|
|
566
|
+
* second, different error about text the author never wrote.
|
|
567
|
+
*/
|
|
568
|
+
function javascriptModuleHarness(moduleUrl) {
|
|
569
|
+
return `${JAVASCRIPT_PRELUDE}
|
|
570
|
+
try {
|
|
571
|
+
${JAVASCRIPT_PAYLOAD}
|
|
572
|
+
const mod = await import(${JSON.stringify(moduleUrl)});
|
|
573
|
+
const exported = typeof mod.default === "function"
|
|
574
|
+
? mod.default
|
|
575
|
+
: typeof mod.transform === "function" ? mod.transform : null;
|
|
576
|
+
if (!exported) {
|
|
577
|
+
const names = Object.keys(mod).filter((key) => key !== "default");
|
|
578
|
+
throw new Error(
|
|
579
|
+
"This transform is a module — it has a top-level \`export\` — so the catalog imported it and " +
|
|
580
|
+
"looked for a function to call. \`export default\` is " + (("default" in mod) ? typeof mod.default : "missing") +
|
|
581
|
+
" and there is no exported \`transform\` function." +
|
|
582
|
+
(names.length > 0 ? " It does export: " + names.join(", ") + "." : "") +
|
|
583
|
+
" Export the function as \`export default\`, or name it \`transform\`."
|
|
584
|
+
);
|
|
585
|
+
}
|
|
586
|
+
const rows = await exported({ records, context });
|
|
587
|
+
process.stdout.write(JSON.stringify({ rows: rows ?? [], logs: captured() }));
|
|
588
|
+
} catch (error) {
|
|
589
|
+
${JAVASCRIPT_FAILURE}
|
|
590
|
+
}
|
|
591
|
+
`;
|
|
592
|
+
}
|
|
593
|
+
/**
|
|
594
|
+
* Capture the console, before any of the author's code can reach it.
|
|
595
|
+
*
|
|
596
|
+
* Shared verbatim by both JavaScript harnesses rather than copied into each: two
|
|
597
|
+
* copies of a log cap are two numbers that drift, and the one that drifts is
|
|
598
|
+
* discovered by a run record nobody can explain.
|
|
599
|
+
*/
|
|
600
|
+
const JAVASCRIPT_PRELUDE = `
|
|
466
601
|
const logs = [];
|
|
467
602
|
let dropped = 0;
|
|
468
603
|
const keep = (line) => {
|
|
@@ -483,8 +618,9 @@ const captured = () => dropped === 0
|
|
|
483
618
|
let input = "";
|
|
484
619
|
process.stdin.setEncoding("utf8");
|
|
485
620
|
for await (const chunk of process.stdin) input += chunk;
|
|
486
|
-
|
|
487
|
-
|
|
621
|
+
`;
|
|
622
|
+
/** Unpack the envelope. `context` is frozen one level down — see below. */
|
|
623
|
+
const JAVASCRIPT_PAYLOAD = `
|
|
488
624
|
const payload = JSON.parse(input || "{}");
|
|
489
625
|
const records = Array.isArray(payload.records) ? payload.records : [];
|
|
490
626
|
// Frozen, and one level down as well, so that a transform assigning to
|
|
@@ -492,20 +628,42 @@ try {
|
|
|
492
628
|
// then confusing whoever reads the next node's code. Nothing propagates out
|
|
493
629
|
// of this process either way; the freeze buys the honest error, not safety.
|
|
494
630
|
const context = Object.freeze({ ...payload.context, env: Object.freeze({ ...payload.context?.env }) });
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
} catch (error) {
|
|
631
|
+
`;
|
|
632
|
+
/** The one JSON line a failed run prints, logs and all. */
|
|
633
|
+
const JAVASCRIPT_FAILURE = `
|
|
499
634
|
process.stdout.write(JSON.stringify({
|
|
500
635
|
error: error instanceof Error ? \`\${error.name}: \${error.message}\` : String(error),
|
|
501
636
|
logs: captured(),
|
|
502
637
|
}));
|
|
503
|
-
}
|
|
504
638
|
`;
|
|
505
|
-
}
|
|
506
639
|
/**
|
|
507
640
|
* The Python harness. `records` in, a list of dicts out.
|
|
508
641
|
*
|
|
642
|
+
* ## Why this did not move to the one-object shape
|
|
643
|
+
*
|
|
644
|
+
* JavaScript moved because a JavaScript transform *states its own signature* —
|
|
645
|
+
* the harness wrote `(records, context)` and the author's code depended on both
|
|
646
|
+
* names being where they were, so a third parameter would have been a change to
|
|
647
|
+
* text nobody was going to re-read. A Python transform states nothing. This
|
|
648
|
+
* harness writes `def transform(records, context):` and indents the author's
|
|
649
|
+
* code into it, so a fourth thing to pass is **one line changed here** and not a
|
|
650
|
+
* single stored transform touched. Python already has the property the object
|
|
651
|
+
* shape was introduced to buy.
|
|
652
|
+
*
|
|
653
|
+
* Moving it anyway would cost the thing the move was for. `records` and
|
|
654
|
+
* `context` are names in scope today; a `payload` dict makes them
|
|
655
|
+
* `payload["records"]` and `payload["context"]`, which is a break in every
|
|
656
|
+
* Python transform in existence — the exact outcome the JavaScript change was
|
|
657
|
+
* designed to avoid. Consistency between the two languages is worth something,
|
|
658
|
+
* but not a migration bought with somebody else's pandas code, and not when the
|
|
659
|
+
* inconsistency is *because* the two languages start from different places.
|
|
660
|
+
*
|
|
661
|
+
* The asymmetry left standing, said out loud: a Python author who writes their
|
|
662
|
+
* own `def transform(...)` at column 0 gets it indented into a nested
|
|
663
|
+
* definition, and the outer `transform` returns `None` — no error, no rows. That
|
|
664
|
+
* is a real footgun and it is older than this change; it is named here so the
|
|
665
|
+
* next person to open this file knows it is known rather than missed.
|
|
666
|
+
*
|
|
509
667
|
* A DataFrame is accepted as a return value and converted, because a transform
|
|
510
668
|
* that reaches for pandas will naturally end with one — making it write
|
|
511
669
|
* `.to_dict("records")` would be a papercut on the only path pandas is worth
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which of the two shapes a JavaScript or TypeScript transform is written in,
|
|
3
|
+
* and the one rule that tells them apart.
|
|
4
|
+
*
|
|
5
|
+
* ## The two shapes
|
|
6
|
+
*
|
|
7
|
+
* A transform used to be — and, for everything already stored, still is — a
|
|
8
|
+
* **bare body**: the text between the braces of a function the harness supplies.
|
|
9
|
+
*
|
|
10
|
+
* ```js
|
|
11
|
+
* return records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }));
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* That shape works, and its problem is not syntax. The harness supplied
|
|
15
|
+
* `(records, context)` **positionally**, so the set of things a transform can be
|
|
16
|
+
* given was fixed by the day the second parameter was added: a third one changes
|
|
17
|
+
* the meaning of every signature ever written, and there is no version of
|
|
18
|
+
* "records, context, andNowAlsoThis" that does not make the previous shape a
|
|
19
|
+
* subset by luck rather than by design. So the supported shape is now a real
|
|
20
|
+
* function over **one object**:
|
|
21
|
+
*
|
|
22
|
+
* ```js
|
|
23
|
+
* export default function transform({ records, context }) {
|
|
24
|
+
* return records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }));
|
|
25
|
+
* }
|
|
26
|
+
* ```
|
|
27
|
+
*
|
|
28
|
+
* A field can be added to that object without touching a single stored
|
|
29
|
+
* transform, which is the entire argument for it.
|
|
30
|
+
*
|
|
31
|
+
* ## The rule
|
|
32
|
+
*
|
|
33
|
+
* **A top-level `export` keyword, and nothing else.** Code that has one is a
|
|
34
|
+
* module; code that has none is a body.
|
|
35
|
+
*
|
|
36
|
+
* That is a discriminator rather than a heuristic, and the reason is worth being
|
|
37
|
+
* exact about: `export` is a *syntax error* inside a function body. Every
|
|
38
|
+
* transform stored today runs as a function body today, so no stored transform
|
|
39
|
+
* can contain a top-level `export` — not "probably does not", cannot. Backward
|
|
40
|
+
* compatibility here is a property of the language, not of how good the guess is.
|
|
41
|
+
*
|
|
42
|
+
* ## What the rule deliberately does not look at
|
|
43
|
+
*
|
|
44
|
+
* Not the word `function`. Not a function *declaration* named `transform`
|
|
45
|
+
* either, which is the tempting second rule and is the one that would break
|
|
46
|
+
* real code:
|
|
47
|
+
*
|
|
48
|
+
* ```js
|
|
49
|
+
* function transform(r) { return { mgmtCd: r["Mgmt Cd"] }; }
|
|
50
|
+
* return records.map(transform);
|
|
51
|
+
* ```
|
|
52
|
+
*
|
|
53
|
+
* That is a bare body which declares a local helper it happens to have named
|
|
54
|
+
* `transform`. A detector that called it the new shape would call the helper
|
|
55
|
+
* with `{records, context}` — one object where a record was expected — and store
|
|
56
|
+
* 100,000 rows of `undefined` without erroring once. Silent wrong data is the
|
|
57
|
+
* worst failure available here, so the detector does not offer an opinion about
|
|
58
|
+
* names at all.
|
|
59
|
+
*
|
|
60
|
+
* ## The scan, and its one known limit
|
|
61
|
+
*
|
|
62
|
+
* Strings, template literals (including `${}` nesting), regular-expression
|
|
63
|
+
* literals and both kinds of comment are skipped, so `// export default` and
|
|
64
|
+
* `"export"` are not exports. The keyword must then appear at **statement
|
|
65
|
+
* position** — start of input, or after `;`, `}`, or a newline — at brace,
|
|
66
|
+
* paren and bracket depth zero.
|
|
67
|
+
*
|
|
68
|
+
* Regular-expression literals are found by the usual rule (a `/` is a regex
|
|
69
|
+
* unless what precedes it could end a value), which is the one place a scanner
|
|
70
|
+
* without a full parser can be wrong. Its consequence is bounded on purpose: a
|
|
71
|
+
* misread makes this return `false` for a module, the code is run as a body, and
|
|
72
|
+
* the author gets `SyntaxError: Unexpected token 'export'` — which
|
|
73
|
+
* {@link transformShapeHint} turns into a sentence naming this exact rule. A
|
|
74
|
+
* wrong answer here produces a reported error, never a silently different run.
|
|
75
|
+
*/
|
|
76
|
+
/** What shape a transform's code is in. */
|
|
77
|
+
export type TransformShape = 'module' | 'body';
|
|
78
|
+
/**
|
|
79
|
+
* Does this code declare an ES module — and so ask to be called as a function
|
|
80
|
+
* over one object?
|
|
81
|
+
*
|
|
82
|
+
* See the module docblock for the rule and why it is the rule. Python is not
|
|
83
|
+
* asked this question: its harness writes the `def` itself, so a Python
|
|
84
|
+
* transform never states a signature and never had the problem this detector
|
|
85
|
+
* exists to solve.
|
|
86
|
+
*/
|
|
87
|
+
export declare function transformDeclaresModule(code: string): boolean;
|
|
88
|
+
/** {@link transformDeclaresModule}, as the shape it names. */
|
|
89
|
+
export declare function transformShape(code: string): TransformShape;
|
|
90
|
+
/**
|
|
91
|
+
* The sentence to add when a body-shaped run died on the one syntax error that
|
|
92
|
+
* means the detector and the author disagreed.
|
|
93
|
+
*
|
|
94
|
+
* The rule above is a scan, not a parser, and its documented limit is that a
|
|
95
|
+
* regular-expression literal read as a division can hide a real `export`. The
|
|
96
|
+
* author then sees `Unexpected token 'export'` from code they believe is a
|
|
97
|
+
* perfectly good module, and has no way to know that a *rule they have never
|
|
98
|
+
* read* is what decided otherwise. Naming the rule in the error is the whole
|
|
99
|
+
* difference between a two-minute fix and an afternoon.
|
|
100
|
+
*
|
|
101
|
+
* Deliberately not a fallback re-run in the other shape. Running code twice
|
|
102
|
+
* because the first attempt failed is guessing with extra steps: the second
|
|
103
|
+
* attempt would report a different error for the same text, and neither error
|
|
104
|
+
* would be trustworthy.
|
|
105
|
+
*/
|
|
106
|
+
export declare function transformShapeHint(shape: TransformShape, stderr: string): string;
|
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Which of the two shapes a JavaScript or TypeScript transform is written in,
|
|
4
|
+
* and the one rule that tells them apart.
|
|
5
|
+
*
|
|
6
|
+
* ## The two shapes
|
|
7
|
+
*
|
|
8
|
+
* A transform used to be — and, for everything already stored, still is — a
|
|
9
|
+
* **bare body**: the text between the braces of a function the harness supplies.
|
|
10
|
+
*
|
|
11
|
+
* ```js
|
|
12
|
+
* return records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }));
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* That shape works, and its problem is not syntax. The harness supplied
|
|
16
|
+
* `(records, context)` **positionally**, so the set of things a transform can be
|
|
17
|
+
* given was fixed by the day the second parameter was added: a third one changes
|
|
18
|
+
* the meaning of every signature ever written, and there is no version of
|
|
19
|
+
* "records, context, andNowAlsoThis" that does not make the previous shape a
|
|
20
|
+
* subset by luck rather than by design. So the supported shape is now a real
|
|
21
|
+
* function over **one object**:
|
|
22
|
+
*
|
|
23
|
+
* ```js
|
|
24
|
+
* export default function transform({ records, context }) {
|
|
25
|
+
* return records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }));
|
|
26
|
+
* }
|
|
27
|
+
* ```
|
|
28
|
+
*
|
|
29
|
+
* A field can be added to that object without touching a single stored
|
|
30
|
+
* transform, which is the entire argument for it.
|
|
31
|
+
*
|
|
32
|
+
* ## The rule
|
|
33
|
+
*
|
|
34
|
+
* **A top-level `export` keyword, and nothing else.** Code that has one is a
|
|
35
|
+
* module; code that has none is a body.
|
|
36
|
+
*
|
|
37
|
+
* That is a discriminator rather than a heuristic, and the reason is worth being
|
|
38
|
+
* exact about: `export` is a *syntax error* inside a function body. Every
|
|
39
|
+
* transform stored today runs as a function body today, so no stored transform
|
|
40
|
+
* can contain a top-level `export` — not "probably does not", cannot. Backward
|
|
41
|
+
* compatibility here is a property of the language, not of how good the guess is.
|
|
42
|
+
*
|
|
43
|
+
* ## What the rule deliberately does not look at
|
|
44
|
+
*
|
|
45
|
+
* Not the word `function`. Not a function *declaration* named `transform`
|
|
46
|
+
* either, which is the tempting second rule and is the one that would break
|
|
47
|
+
* real code:
|
|
48
|
+
*
|
|
49
|
+
* ```js
|
|
50
|
+
* function transform(r) { return { mgmtCd: r["Mgmt Cd"] }; }
|
|
51
|
+
* return records.map(transform);
|
|
52
|
+
* ```
|
|
53
|
+
*
|
|
54
|
+
* That is a bare body which declares a local helper it happens to have named
|
|
55
|
+
* `transform`. A detector that called it the new shape would call the helper
|
|
56
|
+
* with `{records, context}` — one object where a record was expected — and store
|
|
57
|
+
* 100,000 rows of `undefined` without erroring once. Silent wrong data is the
|
|
58
|
+
* worst failure available here, so the detector does not offer an opinion about
|
|
59
|
+
* names at all.
|
|
60
|
+
*
|
|
61
|
+
* ## The scan, and its one known limit
|
|
62
|
+
*
|
|
63
|
+
* Strings, template literals (including `${}` nesting), regular-expression
|
|
64
|
+
* literals and both kinds of comment are skipped, so `// export default` and
|
|
65
|
+
* `"export"` are not exports. The keyword must then appear at **statement
|
|
66
|
+
* position** — start of input, or after `;`, `}`, or a newline — at brace,
|
|
67
|
+
* paren and bracket depth zero.
|
|
68
|
+
*
|
|
69
|
+
* Regular-expression literals are found by the usual rule (a `/` is a regex
|
|
70
|
+
* unless what precedes it could end a value), which is the one place a scanner
|
|
71
|
+
* without a full parser can be wrong. Its consequence is bounded on purpose: a
|
|
72
|
+
* misread makes this return `false` for a module, the code is run as a body, and
|
|
73
|
+
* the author gets `SyntaxError: Unexpected token 'export'` — which
|
|
74
|
+
* {@link transformShapeHint} turns into a sentence naming this exact rule. A
|
|
75
|
+
* wrong answer here produces a reported error, never a silently different run.
|
|
76
|
+
*/
|
|
77
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
78
|
+
exports.transformDeclaresModule = transformDeclaresModule;
|
|
79
|
+
exports.transformShape = transformShape;
|
|
80
|
+
exports.transformShapeHint = transformShapeHint;
|
|
81
|
+
/** Characters that may make up an identifier, for the boundary check. */
|
|
82
|
+
function isIdentifierChar(char) {
|
|
83
|
+
return /[\p{ID_Continue}$]/u.test(char);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Words after which a `/` is a regular expression even though the character
|
|
87
|
+
* before it is an identifier character.
|
|
88
|
+
*
|
|
89
|
+
* `return /a'b/` is the case that matters: without this the apostrophe would
|
|
90
|
+
* open a string literal that never closes, and everything after it — including
|
|
91
|
+
* a real `export` — would be skipped as string contents. That is precisely the
|
|
92
|
+
* scanner's documented failure mode, so the cheap fix for its likeliest cause is
|
|
93
|
+
* worth the fifteen words.
|
|
94
|
+
*/
|
|
95
|
+
const REGEX_PRECEDING_KEYWORDS = new Set([
|
|
96
|
+
'await',
|
|
97
|
+
'case',
|
|
98
|
+
'delete',
|
|
99
|
+
'do',
|
|
100
|
+
'else',
|
|
101
|
+
'in',
|
|
102
|
+
'instanceof',
|
|
103
|
+
'new',
|
|
104
|
+
'of',
|
|
105
|
+
'return',
|
|
106
|
+
'throw',
|
|
107
|
+
'typeof',
|
|
108
|
+
'void',
|
|
109
|
+
'yield',
|
|
110
|
+
]);
|
|
111
|
+
/**
|
|
112
|
+
* Whether a `/` at this point starts a regular expression rather than a
|
|
113
|
+
* division.
|
|
114
|
+
*
|
|
115
|
+
* The conventional rule: a regex may only appear where a *value* may appear, so
|
|
116
|
+
* anything that could end a value — an identifier, a number, a closing bracket
|
|
117
|
+
* or paren — means division. `}` is treated as ending a value too, which is the
|
|
118
|
+
* common convention and wrong only for `if (x) {} /re/.test(y)`, a statement
|
|
119
|
+
* nobody writes.
|
|
120
|
+
*/
|
|
121
|
+
function startsRegex(code, at, lastSignificant) {
|
|
122
|
+
if (lastSignificant === '')
|
|
123
|
+
return true;
|
|
124
|
+
if (lastSignificant === ')' || lastSignificant === ']' || lastSignificant === '}')
|
|
125
|
+
return false;
|
|
126
|
+
if (!isIdentifierChar(lastSignificant))
|
|
127
|
+
return true;
|
|
128
|
+
// Back over the whitespace between the word and the slash, then over the word
|
|
129
|
+
// itself: `return /x/` puts a space where `code[at - 1]` is.
|
|
130
|
+
let end = at;
|
|
131
|
+
while (end > 0 && /\s/.test(code[end - 1]))
|
|
132
|
+
end -= 1;
|
|
133
|
+
let start = end;
|
|
134
|
+
while (start > 0 && isIdentifierChar(code[start - 1]))
|
|
135
|
+
start -= 1;
|
|
136
|
+
return REGEX_PRECEDING_KEYWORDS.has(code.slice(start, end));
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* The scan itself, as a cursor over the source.
|
|
140
|
+
*
|
|
141
|
+
* A class rather than one long loop because the loop *was* one long loop, and it
|
|
142
|
+
* scored 126 on a complexity budget of 15 — which in this case the linter was
|
|
143
|
+
* right about. Every branch below is one lexical thing the scanner has to be
|
|
144
|
+
* able to walk past without losing its place, and naming them separately is what
|
|
145
|
+
* makes it possible to read whether the list is complete.
|
|
146
|
+
*
|
|
147
|
+
* State is deliberately minimal: where we are, what the last meaningful
|
|
148
|
+
* character was, whether a newline has happened since, and three depths plus a
|
|
149
|
+
* stack of open template literals. Nothing here builds a tree, because nothing
|
|
150
|
+
* here needs to answer any question except one.
|
|
151
|
+
*/
|
|
152
|
+
class ModuleScanner {
|
|
153
|
+
code;
|
|
154
|
+
i = 0;
|
|
155
|
+
lastSignificant = '';
|
|
156
|
+
newlineSince = false;
|
|
157
|
+
braces = 0;
|
|
158
|
+
parens = 0;
|
|
159
|
+
brackets = 0;
|
|
160
|
+
/**
|
|
161
|
+
* Brace depths at which template literals opened, innermost last. A `}` ends
|
|
162
|
+
* an interpolation when the depth has come back to the one recorded for it.
|
|
163
|
+
*/
|
|
164
|
+
templates = [];
|
|
165
|
+
constructor(code) {
|
|
166
|
+
this.code = code;
|
|
167
|
+
}
|
|
168
|
+
/** Whether a top-level `export` appears anywhere in the source. */
|
|
169
|
+
findsExport() {
|
|
170
|
+
while (this.i < this.code.length) {
|
|
171
|
+
if (this.step())
|
|
172
|
+
return true;
|
|
173
|
+
this.i += 1;
|
|
174
|
+
}
|
|
175
|
+
return false;
|
|
176
|
+
}
|
|
177
|
+
step() {
|
|
178
|
+
const char = this.code[this.i];
|
|
179
|
+
if (this.skipTrivia(char))
|
|
180
|
+
return false;
|
|
181
|
+
if (this.skipLiteral(char))
|
|
182
|
+
return false;
|
|
183
|
+
if (this.closesInterpolation(char))
|
|
184
|
+
return false;
|
|
185
|
+
this.adjustDepth(char);
|
|
186
|
+
if (this.isExportHere(char))
|
|
187
|
+
return true;
|
|
188
|
+
this.lastSignificant = char;
|
|
189
|
+
this.newlineSince = false;
|
|
190
|
+
return false;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Whitespace and comments.
|
|
194
|
+
*
|
|
195
|
+
* A comment leaves {@link lastSignificant} alone on purpose: a comment between
|
|
196
|
+
* a `;` and an `export` does not move the `export` off the start of a
|
|
197
|
+
* statement, and a rule that thought it did would fail on the most ordinary
|
|
198
|
+
* thing anybody writes above a function.
|
|
199
|
+
*/
|
|
200
|
+
skipTrivia(char) {
|
|
201
|
+
if (char === '\n') {
|
|
202
|
+
this.newlineSince = true;
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
if (char === ' ' || char === '\t' || char === '\r')
|
|
206
|
+
return true;
|
|
207
|
+
return this.skipComment(char);
|
|
208
|
+
}
|
|
209
|
+
skipComment(char) {
|
|
210
|
+
if (char !== '/')
|
|
211
|
+
return false;
|
|
212
|
+
const next = this.code[this.i + 1];
|
|
213
|
+
if (next === '/') {
|
|
214
|
+
while (this.i < this.code.length && this.code[this.i] !== '\n')
|
|
215
|
+
this.i += 1;
|
|
216
|
+
this.newlineSince = true;
|
|
217
|
+
return true;
|
|
218
|
+
}
|
|
219
|
+
if (next !== '*')
|
|
220
|
+
return false;
|
|
221
|
+
const end = this.code.indexOf('*/', this.i + 2);
|
|
222
|
+
const chunk = end === -1 ? this.code.slice(this.i) : this.code.slice(this.i, end + 2);
|
|
223
|
+
// A block comment spanning lines ends the line for the newline rule, and an
|
|
224
|
+
// unterminated one swallows the rest of the file.
|
|
225
|
+
if (chunk.includes('\n'))
|
|
226
|
+
this.newlineSince = true;
|
|
227
|
+
this.i = end === -1 ? this.code.length : end + 1;
|
|
228
|
+
return true;
|
|
229
|
+
}
|
|
230
|
+
/** Strings, template literals and regular expressions — walked past whole. */
|
|
231
|
+
skipLiteral(char) {
|
|
232
|
+
if (char === '"' || char === "'") {
|
|
233
|
+
this.skipQuoted(char);
|
|
234
|
+
return this.consumedValue(char);
|
|
235
|
+
}
|
|
236
|
+
if (char === '`') {
|
|
237
|
+
this.openTemplate();
|
|
238
|
+
return this.consumedValue('`');
|
|
239
|
+
}
|
|
240
|
+
if (char === '/' && startsRegex(this.code, this.i, this.lastSignificant)) {
|
|
241
|
+
this.skipRegex();
|
|
242
|
+
return this.consumedValue('/');
|
|
243
|
+
}
|
|
244
|
+
return false;
|
|
245
|
+
}
|
|
246
|
+
/** Record that something which can end a value has just been walked past. */
|
|
247
|
+
consumedValue(ending) {
|
|
248
|
+
this.lastSignificant = ending;
|
|
249
|
+
this.newlineSince = false;
|
|
250
|
+
return true;
|
|
251
|
+
}
|
|
252
|
+
skipQuoted(quote) {
|
|
253
|
+
this.i += 1;
|
|
254
|
+
while (this.i < this.code.length && this.code[this.i] !== quote) {
|
|
255
|
+
if (this.code[this.i] === '\\')
|
|
256
|
+
this.i += 1;
|
|
257
|
+
this.i += 1;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
skipRegex() {
|
|
261
|
+
this.i += 1;
|
|
262
|
+
let inClass = false;
|
|
263
|
+
while (this.i < this.code.length) {
|
|
264
|
+
const char = this.code[this.i];
|
|
265
|
+
if (char === '\\')
|
|
266
|
+
this.i += 1;
|
|
267
|
+
else if (char === '[')
|
|
268
|
+
inClass = true;
|
|
269
|
+
else if (char === ']')
|
|
270
|
+
inClass = false;
|
|
271
|
+
// A newline ends it either way: an unterminated regex is a syntax error,
|
|
272
|
+
// and running to the end of the file on one would hide everything after.
|
|
273
|
+
else if (char === '\n' || (char === '/' && !inClass))
|
|
274
|
+
break;
|
|
275
|
+
this.i += 1;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Walk a template literal to its closing backtick, or to the `${` that hands
|
|
280
|
+
* control back to the code scanner.
|
|
281
|
+
*/
|
|
282
|
+
openTemplate() {
|
|
283
|
+
this.templates.push(this.braces);
|
|
284
|
+
this.i += 1;
|
|
285
|
+
while (this.i < this.code.length) {
|
|
286
|
+
const char = this.code[this.i];
|
|
287
|
+
if (char === '\\') {
|
|
288
|
+
this.i += 1;
|
|
289
|
+
}
|
|
290
|
+
else if (char === '`') {
|
|
291
|
+
this.templates.pop();
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
else if (char === '$' && this.code[this.i + 1] === '{') {
|
|
295
|
+
this.i += 1;
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
this.i += 1;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* A `}` that ends an interpolation rather than a block: keep reading the
|
|
303
|
+
* template literal it was inside.
|
|
304
|
+
*/
|
|
305
|
+
closesInterpolation(char) {
|
|
306
|
+
if (char !== '}')
|
|
307
|
+
return false;
|
|
308
|
+
if (this.templates.length === 0)
|
|
309
|
+
return false;
|
|
310
|
+
if (this.templates[this.templates.length - 1] !== this.braces)
|
|
311
|
+
return false;
|
|
312
|
+
this.templates.pop();
|
|
313
|
+
this.i += 1;
|
|
314
|
+
this.resumeTemplate();
|
|
315
|
+
return true;
|
|
316
|
+
}
|
|
317
|
+
resumeTemplate() {
|
|
318
|
+
while (this.i < this.code.length) {
|
|
319
|
+
const char = this.code[this.i];
|
|
320
|
+
if (char === '\\') {
|
|
321
|
+
this.i += 1;
|
|
322
|
+
}
|
|
323
|
+
else if (char === '`') {
|
|
324
|
+
this.consumedValue('`');
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
else if (char === '$' && this.code[this.i + 1] === '{') {
|
|
328
|
+
this.templates.push(this.braces);
|
|
329
|
+
this.i += 1;
|
|
330
|
+
this.newlineSince = false;
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
this.i += 1;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
adjustDepth(char) {
|
|
337
|
+
if (char === '{')
|
|
338
|
+
this.braces += 1;
|
|
339
|
+
else if (char === '}')
|
|
340
|
+
this.braces -= 1;
|
|
341
|
+
else if (char === '[')
|
|
342
|
+
this.brackets += 1;
|
|
343
|
+
else if (char === ']')
|
|
344
|
+
this.brackets -= 1;
|
|
345
|
+
else if (char === '(')
|
|
346
|
+
this.parens += 1;
|
|
347
|
+
else if (char === ')')
|
|
348
|
+
this.parens -= 1;
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* The whole question, in one place: the keyword `export`, whole, at the start
|
|
352
|
+
* of a statement, at the outermost level of the source.
|
|
353
|
+
*/
|
|
354
|
+
isExportHere(char) {
|
|
355
|
+
if (char !== 'e')
|
|
356
|
+
return false;
|
|
357
|
+
if (this.braces !== 0 || this.parens !== 0 || this.brackets !== 0)
|
|
358
|
+
return false;
|
|
359
|
+
if (this.templates.length > 0)
|
|
360
|
+
return false;
|
|
361
|
+
if (!this.atStatementStart())
|
|
362
|
+
return false;
|
|
363
|
+
if (!this.code.startsWith('export', this.i))
|
|
364
|
+
return false;
|
|
365
|
+
return !isIdentifierChar(this.code[this.i + 6] ?? '');
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Start of input, after a `;` or a `}`, or on a new line — the last because
|
|
369
|
+
* JavaScript does not require the semicolon and plenty of people do not write
|
|
370
|
+
* one.
|
|
371
|
+
*/
|
|
372
|
+
atStatementStart() {
|
|
373
|
+
return (this.lastSignificant === '' ||
|
|
374
|
+
this.lastSignificant === ';' ||
|
|
375
|
+
this.lastSignificant === '}' ||
|
|
376
|
+
this.newlineSince);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
/**
|
|
380
|
+
* Does this code declare an ES module — and so ask to be called as a function
|
|
381
|
+
* over one object?
|
|
382
|
+
*
|
|
383
|
+
* See the module docblock for the rule and why it is the rule. Python is not
|
|
384
|
+
* asked this question: its harness writes the `def` itself, so a Python
|
|
385
|
+
* transform never states a signature and never had the problem this detector
|
|
386
|
+
* exists to solve.
|
|
387
|
+
*/
|
|
388
|
+
function transformDeclaresModule(code) {
|
|
389
|
+
return new ModuleScanner(code).findsExport();
|
|
390
|
+
}
|
|
391
|
+
/** {@link transformDeclaresModule}, as the shape it names. */
|
|
392
|
+
function transformShape(code) {
|
|
393
|
+
return transformDeclaresModule(code) ? 'module' : 'body';
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* The sentence to add when a body-shaped run died on the one syntax error that
|
|
397
|
+
* means the detector and the author disagreed.
|
|
398
|
+
*
|
|
399
|
+
* The rule above is a scan, not a parser, and its documented limit is that a
|
|
400
|
+
* regular-expression literal read as a division can hide a real `export`. The
|
|
401
|
+
* author then sees `Unexpected token 'export'` from code they believe is a
|
|
402
|
+
* perfectly good module, and has no way to know that a *rule they have never
|
|
403
|
+
* read* is what decided otherwise. Naming the rule in the error is the whole
|
|
404
|
+
* difference between a two-minute fix and an afternoon.
|
|
405
|
+
*
|
|
406
|
+
* Deliberately not a fallback re-run in the other shape. Running code twice
|
|
407
|
+
* because the first attempt failed is guessing with extra steps: the second
|
|
408
|
+
* attempt would report a different error for the same text, and neither error
|
|
409
|
+
* would be trustworthy.
|
|
410
|
+
*/
|
|
411
|
+
function transformShapeHint(shape, stderr) {
|
|
412
|
+
if (shape !== 'body')
|
|
413
|
+
return '';
|
|
414
|
+
if (!/Unexpected token '?export/.test(stderr))
|
|
415
|
+
return '';
|
|
416
|
+
return ('\nThis ran as a bare function body, because the catalog looks for an `export` keyword at ' +
|
|
417
|
+
'the start of a statement, outside any brackets, string or comment — and found none. If ' +
|
|
418
|
+
'this was meant to be a module, move the `export` to the start of its own line.');
|
|
419
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dudousxd/nestjs-catalog",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.22.0",
|
|
4
4
|
"description": "A metadata registry for NestJS: object types, properties and relations, derived from your ORM and enriched with decorators.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Davide Carvalho",
|