@dudousxd/nestjs-catalog 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. package/dist/catalog.controller.js +36 -2
  2. package/dist/catalog.pipeline.d.ts +156 -13
  3. package/dist/catalog.pipeline.js +92 -2
  4. package/dist/catalog.secrets.d.ts +283 -0
  5. package/dist/catalog.secrets.js +209 -0
  6. package/dist/catalog.workspace.d.ts +154 -0
  7. package/dist/catalog.workspace.js +59 -1
  8. package/dist/client.d.ts +13 -2
  9. package/dist/client.js +23 -1
  10. package/dist/index.d.ts +3 -2
  11. package/dist/index.js +15 -2
  12. package/dist/transform-runner.d.ts +22 -5
  13. package/dist/transform-runner.js +126 -9
  14. package/package.json +1 -1
package/dist/catalog.workspace.js CHANGED
@@ -9,12 +9,60 @@
9
9
  * mount the catalog without any of this and notice nothing missing.
10
10
  */
11
11
  Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.CATALOG_WORKSPACE_STORE = exports.CATALOG_TRACE_STORE = exports.CATALOG_TRACE_OUTCOMES = void 0;
12
+ exports.CATALOG_WORKSPACE_STORE = exports.CATALOG_TRACE_STORE = exports.CATALOG_TRACE_OUTCOMES = exports.CATALOG_REVISION_LIMIT = void 0;
13
13
  exports.isCatalogTraceOutcome = isCatalogTraceOutcome;
14
14
  exports.traceOutcomeFilter = traceOutcomeFilter;
15
15
  exports.isTraceStore = isTraceStore;
16
16
  exports.embeddedVisualization = embeddedVisualization;
17
+ exports.supportsSavedQueryRevisions = supportsSavedQueryRevisions;
17
18
  exports.isWorkspaceStore = isWorkspaceStore;
19
+ /**
20
+ * How many revisions are kept per subject. Writing a newer one drops the oldest
21
+ * beyond this.
22
+ *
23
+ * ## Why there is a cap at all
24
+ *
25
+ * This is append-only text that grows forever, and it is the fourth append-only
26
+ * table in the bundled store. The other three earn their unboundedness and this
27
+ * one does not. An audit event and a connector run are each one small row per
28
+ * *thing that happened*, at a rate an operator can read off their own load
29
+ * schedule; staged rows are dropped the moment the run that produced them is
30
+ * finished with. A revision is neither: it grows with how often somebody edits,
31
+ * which nobody meters, and every row carries a whole code body rather than a
32
+ * counter. Unpredictable in rate *and* large per row is the combination worth
33
+ * bounding — and "this grows; here is the query to prune it" would have been a
34
+ * fourth unbounded table with a paragraph in front of it.
35
+ *
36
+ * ## Why a count per subject rather than an age bound
37
+ *
38
+ * An age bound keys the wrong thing. A transform edited twice in 2019 and relied
39
+ * on ever since would lose both revisions, while one edited daily keeps
40
+ * everything — exactly backwards. What makes a revision unrecoverable is being
41
+ * superseded, so that is what the bound counts.
42
+ *
43
+ * ## What it costs, stated rather than implied
44
+ *
45
+ * A run's `transformVersion` can name a revision that has been evicted: a
46
+ * transform saved more than this many times can no longer produce its earliest
47
+ * code. That loss is real. It is strictly smaller than the one it replaces —
48
+ * where every version but the newest was unrecoverable — and it is visible
49
+ * rather than silent, because a caller holding a version older than the oldest
50
+ * revision in the list can see that the list does not reach that far.
51
+ *
52
+ * At the cap a 4 KB body costs 200 KB per subject, so a thousand heavily-edited
53
+ * subjects cost roughly 200 MB. That is a ceiling, which is the point of having
54
+ * one.
55
+ *
56
+ * ## Why a constant and not a module option
57
+ *
58
+ * The number is part of what this table promises, and a console should be able
59
+ * to print "the last 50 are kept" without a round trip to ask which deployment
60
+ * it is talking to. A knob is also a promise to support every value of it,
61
+ * including the one that switches the feature off and is then reported as a bug.
62
+ * It becomes an option on the day there is a deployment it is wrong for, rather
63
+ * than in anticipation of one.
64
+ */
65
+ exports.CATALOG_REVISION_LIMIT = 50;
18
66
  /**
19
67
  * The same events, told as stories instead of as a list.
20
68
  *
@@ -102,6 +150,16 @@ function embeddedVisualization(saved, cardLibrary) {
102
150
  const base = saved ?? { kind: 'table' };
103
151
  return cardLibrary ? { ...base, library: cardLibrary } : base;
104
152
  }
153
+ /**
154
+ * Whether this store keeps a saved query's history.
155
+ *
156
+ * Checks the method rather than a flag, the same way {@link isWorkspaceStore}
157
+ * and `supportsWorkflows` do: a flag is a claim and a method is the thing
158
+ * itself.
159
+ */
160
+ function supportsSavedQueryRevisions(store) {
161
+ return typeof store.listSavedQueryRevisions === 'function';
162
+ }
105
163
  function isWorkspaceStore(store) {
106
164
  return (typeof store === 'object' &&
107
165
  store !== null &&
package/dist/client.d.ts CHANGED
@@ -9,7 +9,8 @@
9
9
  * own UI. The endpoints alone are not an API; the endpoints plus the response
10
10
  * types are.
11
11
  */
12
- export type { AuditQuery, CatalogAuditEvent, Dashboard, DashboardCard, QueryVisualization, SaveQueryInput, SavedQuery, } from './catalog.workspace';
12
+ export type { AuditQuery, CatalogAuditEvent, CatalogRevision, Dashboard, DashboardCard, QueryVisualization, SaveQueryInput, SavedQuery, } from './catalog.workspace';
13
+ export { CATALOG_REVISION_LIMIT } from './catalog.workspace';
13
14
  export type { CatalogQueryRelation, CatalogQueryRequest, CatalogQueryResult, } from './catalog.query';
14
15
  export type { CatalogSearchField, CatalogSearchHit, CatalogSearchKind, CatalogSearchRank, CatalogSearchResult, } from './search.types';
15
16
  export type { CatalogGraph, CatalogObjectPage, CatalogObjectQuery, CatalogObjectTypeDef, CatalogOverlay, CatalogPropertyDef, CatalogRelationDef, CatalogSnapshot, RelationKind, ScalarType, } from './catalog.types';
@@ -66,6 +67,15 @@ export declare const catalogRoutes: {
66
67
  readonly workspaceCapabilities: () => string;
67
68
  readonly savedQueries: () => string;
68
69
  readonly savedQuery: (id: string) => string;
70
+ /**
71
+ * Every SQL this query has ever been.
72
+ *
73
+ * A sub-resource of the saved query rather than a `?version=` on it, because
74
+ * the question a diff screen asks first is "what were all of them" — it has to
75
+ * see the list before it knows which two to compare, and one request that
76
+ * answers that beats a list plus two fetches.
77
+ */
78
+ readonly savedQueryRevisions: (id: string) => string;
69
79
  readonly runSavedQuery: (id: string) => string;
70
80
  readonly exportSavedQuery: (id: string) => string;
71
81
  readonly dashboards: () => string;
@@ -93,5 +103,6 @@ export { CONNECTOR_KINDS, isConnectorKind, isTransformLanguage, isWorkflowEdge,
93
103
  * `WORKFLOW_NODE_ID_PATTERN` and `WORKFLOW_NODE_KINDS` are what a palette and an
94
104
  * id field should be built from rather than from a second copy that drifts.
95
105
  */
96
- export { validateWorkflow, WORKFLOW_EXECUTION_MODES, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, workflowGraphHash, workflowRunOrder, isWorkflowExecutionMode, isWorkflowNodeKind, } from './catalog.pipeline';
106
+ export { validateWorkflow, WORKFLOW_EXECUTION_MODES, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_STATUSES, workflowGraphHash, workflowRunOrder, isWorkflowExecutionMode, isWorkflowNodeKind, isWorkflowStatus, } from './catalog.pipeline';
107
+ export type { WorkflowStatus } from './catalog.pipeline';
97
108
  export type { CatalogTrace, CatalogTraceList, CatalogTraceOutcome, CatalogTraceSpan, TraceQuery, } from './catalog.workspace';
package/dist/client.js CHANGED
@@ -11,7 +11,12 @@
11
11
  * types are.
12
12
  */
13
13
  Object.defineProperty(exports, "__esModule", { value: true });
14
- exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.TRANSFORM_LANGUAGES = exports.isWorkflowNode = exports.isWorkflowEdge = exports.isTransformLanguage = exports.isConnectorKind = exports.CONNECTOR_KINDS = exports.catalogRoutes = void 0;
14
+ exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.TRANSFORM_LANGUAGES = exports.isWorkflowNode = exports.isWorkflowEdge = exports.isTransformLanguage = exports.isConnectorKind = exports.CONNECTOR_KINDS = exports.catalogRoutes = exports.CATALOG_REVISION_LIMIT = void 0;
15
+ // A value, not a type: a screen saying how far back the history goes should read
16
+ // the number rather than print one of its own. See its docblock for what the cap
17
+ // costs.
18
+ var catalog_workspace_1 = require("./catalog.workspace");
19
+ Object.defineProperty(exports, "CATALOG_REVISION_LIMIT", { enumerable: true, get: function () { return catalog_workspace_1.CATALOG_REVISION_LIMIT; } });
15
20
  /**
16
21
  * Builds the paths the catalog controller serves, relative to wherever it was
17
22
  * mounted. Kept as string builders rather than a fetch wrapper so the host
@@ -40,6 +45,15 @@ exports.catalogRoutes = {
40
45
  workspaceCapabilities: () => '/catalog/workspace/capabilities',
41
46
  savedQueries: () => '/catalog/saved-queries',
42
47
  savedQuery: (id) => `/catalog/saved-queries/${encodeURIComponent(id)}`,
48
+ /**
49
+ * Every SQL this query has ever been.
50
+ *
51
+ * A sub-resource of the saved query rather than a `?version=` on it, because
52
+ * the question a diff screen asks first is "what were all of them" — it has to
53
+ * see the list before it knows which two to compare, and one request that
54
+ * answers that beats a list plus two fetches.
55
+ */
56
+ savedQueryRevisions: (id) => `/catalog/saved-queries/${encodeURIComponent(id)}/revisions`,
43
57
  runSavedQuery: (id) => `/catalog/saved-queries/${encodeURIComponent(id)}/run`,
44
58
  exportSavedQuery: (id) => `/catalog/saved-queries/${encodeURIComponent(id)}/export.csv`,
45
59
  dashboards: () => '/catalog/dashboards',
@@ -87,7 +101,15 @@ Object.defineProperty(exports, "WORKFLOW_EXECUTION_MODES", { enumerable: true, g
87
101
  Object.defineProperty(exports, "WORKFLOW_ISSUE_CODES", { enumerable: true, get: function () { return catalog_pipeline_2.WORKFLOW_ISSUE_CODES; } });
88
102
  Object.defineProperty(exports, "WORKFLOW_NODE_ID_PATTERN", { enumerable: true, get: function () { return catalog_pipeline_2.WORKFLOW_NODE_ID_PATTERN; } });
89
103
  Object.defineProperty(exports, "WORKFLOW_NODE_KINDS", { enumerable: true, get: function () { return catalog_pipeline_2.WORKFLOW_NODE_KINDS; } });
104
+ // The draft/ready pair, for the same reason as the list above: a canvas that
105
+ // cannot see it restates it, and the copy is what drifts. Without this the
106
+ // editor could not tell a graph it is allowed to store from one the server
107
+ // would refuse — so it told everybody the second, which is wrong for every
108
+ // draft and is exactly the kind of confident-and-false sentence this codebase
109
+ // keeps removing.
110
+ Object.defineProperty(exports, "WORKFLOW_STATUSES", { enumerable: true, get: function () { return catalog_pipeline_2.WORKFLOW_STATUSES; } });
90
111
  Object.defineProperty(exports, "workflowGraphHash", { enumerable: true, get: function () { return catalog_pipeline_2.workflowGraphHash; } });
91
112
  Object.defineProperty(exports, "workflowRunOrder", { enumerable: true, get: function () { return catalog_pipeline_2.workflowRunOrder; } });
92
113
  Object.defineProperty(exports, "isWorkflowExecutionMode", { enumerable: true, get: function () { return catalog_pipeline_2.isWorkflowExecutionMode; } });
93
114
  Object.defineProperty(exports, "isWorkflowNodeKind", { enumerable: true, get: function () { return catalog_pipeline_2.isWorkflowNodeKind; } });
115
+ Object.defineProperty(exports, "isWorkflowStatus", { enumerable: true, get: function () { return catalog_pipeline_2.isWorkflowStatus; } });
package/dist/index.d.ts CHANGED
@@ -3,18 +3,19 @@ export { CATALOG_EVENT_PHASE, CATALOG_EVENT_PHASE_FALLBACK, CATALOG_EVENTS, CATA
3
3
  export { CatalogModule } from './catalog.module';
4
4
  export { assertReadOnlyShape, type CatalogQueryRelation, type CatalogQueryRequest, type CatalogQueryResult, type CatalogQueryStore, isQueryStore, } from './catalog.query';
5
5
  export { CATALOG_OPTIONS, type CatalogModuleOptions } from './catalog.options';
6
+ export * from './catalog.secrets';
6
7
  export { type CatalogOverlayStore, FileCatalogOverlayStore, InMemoryCatalogOverlayStore, } from './catalog.overlay-store';
7
8
  export { CATALOG_OVERLAY_STORE } from './catalog.overlay-store.token';
8
9
  export { MikroOrmCatalogRegistry } from './catalog.registry';
9
10
  export { CatalogRegistry } from './catalog.registry.base';
10
- export { CATALOG_PIPELINE_STORE, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogPipelineStore, type CatalogStageStore, type CatalogTransform, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowStore, type ConnectorKind, type ConnectorRun, isConnectorKind, isPipelineStore, isTransformLanguage, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowNode, isWorkflowNodeKind, supportsWorkflows, supportsWorkflowStages, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, type TransformLanguage, type TransformResult, type TransformRunner, validateWorkflow, WORKFLOW_EXECUTION_MODES, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowGraph, workflowGraphHash, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, type WorkflowNodeOutcome, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowSinkNode, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
11
+ export { CATALOG_PIPELINE_STORE, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogPipelineStore, type CatalogStageStore, type CatalogTransform, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowStore, type ConnectorKind, type ConnectorRun, isConnectorKind, isPipelineStore, isTransformLanguage, supportsTransformRevisions, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowNode, isWorkflowNodeKind, isWorkflowStatus, supportsWorkflows, supportsWorkflowStages, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, type TransformLanguage, type TransformResult, type TransformRunner, validateWorkflow, WORKFLOW_EXECUTION_MODES, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_STATUSES, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowGraph, workflowGraphHash, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, type WorkflowNodeOutcome, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowSinkNode, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowStatus, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
11
12
  export * from './catalog.environment';
12
13
  export { QueryCache, toCsv } from './catalog.query-cache';
13
14
  export { SubprocessTransformRunner, type TransformRunnerOptions, } from './transform-runner';
14
15
  export { CatalogService } from './catalog.service';
15
16
  export { DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT, bestMatch, emptySearch, maySearch, type SearchInput, type SearchableDashboard, type SearchableSavedQuery, searchCatalog, visibleToPrincipal, } from './search';
16
17
  export type { CatalogSearchField, CatalogSearchHit, CatalogSearchKind, CatalogSearchRank, CatalogSearchResult, } from './search.types';
17
- export { type AuditQuery, CATALOG_TRACE_OUTCOMES, CATALOG_TRACE_STORE, CATALOG_WORKSPACE_STORE, type CatalogAuditEvent, type CatalogTrace, type CatalogTraceList, type CatalogTraceOutcome, type CatalogTraceSpan, type CatalogTraceStore, type CatalogTraceTotals, type CatalogUnlinkedList, type CatalogWorkspaceStore, type Dashboard, type DashboardCard, type EmbeddedChart, type EmbeddedChartPlacement, type EmbeddedDashboard, embeddedVisualization, isCatalogTraceOutcome, isTraceStore, isWorkspaceStore, type QueryVisualization, type SaveQueryInput, type SavedQuery, type TraceQuery, traceOutcomeFilter, } from './catalog.workspace';
18
+ export { type AuditQuery, CATALOG_REVISION_LIMIT, CATALOG_TRACE_OUTCOMES, CATALOG_TRACE_STORE, CATALOG_WORKSPACE_STORE, type CatalogAuditEvent, type CatalogRevision, type CatalogTrace, type CatalogTraceList, type CatalogTraceOutcome, type CatalogTraceSpan, type CatalogTraceStore, type CatalogTraceTotals, type CatalogUnlinkedList, type CatalogWorkspaceStore, type Dashboard, type DashboardCard, type EmbeddedChart, type EmbeddedChartPlacement, type EmbeddedDashboard, embeddedVisualization, isCatalogTraceOutcome, isTraceStore, isWorkspaceStore, type QueryVisualization, type SaveQueryInput, type SavedQuery, supportsSavedQueryRevisions, type TraceQuery, traceOutcomeFilter, } from './catalog.workspace';
18
19
  export { CATALOG_PRINCIPAL_RESOLVER, type CatalogActor, type CatalogGrants, type CatalogPrincipal, type CatalogPrincipalResolver, type CatalogScope, composePrincipalId, delegatePrincipal, expandScopes, hasScope, parsePrincipalId, PRINCIPAL_ACTOR_SEPARATOR, maySeeClassification, mayRead, mayWrite, readableObjectPage, StaticKeyPrincipalResolver, } from './catalog.principal';
19
20
  export * from './catalog.access';
20
21
  export { assertNoColumnCollisions, CATALOG_RESERVED_COLUMNS, CATALOG_SNAPSHOT_MODES, CATALOG_STORE, type CarryForwardResult, type CatalogColumnCollision, CatalogColumnCollisionError, type CatalogMergeStore, type CatalogReadQuery, type CatalogReadResult, type CatalogReadStore, type CatalogReservedColumn, type CatalogSnapshotMode, type CatalogStoreCapabilities, type CatalogWriteStore, type ColumnCollisionOptions, findColumnCollisions, isCatalogStoreCapabilities, isReservedColumn, isWriteStore, type SnapshotRef, supportsCarryForward, } from './catalog.store';
package/dist/index.js CHANGED
@@ -14,8 +14,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.searchCatalog = exports.maySearch = exports.emptySearch = exports.bestMatch = exports.MAX_SEARCH_LIMIT = exports.DEFAULT_SEARCH_LIMIT = exports.CatalogService = exports.SubprocessTransformRunner = exports.toCsv = exports.QueryCache = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.TRANSFORM_LANGUAGES = exports.TRANSFORM_RUNNER = exports.supportsWorkflowStages = exports.supportsWorkflows = exports.isWorkflowNodeKind = exports.isWorkflowNode = exports.isWorkflowExecutionMode = exports.isWorkflowEdge = exports.isTransformLanguage = exports.isPipelineStore = exports.isConnectorKind = exports.CONNECTOR_KINDS = exports.CATALOG_PIPELINE_STORE = exports.CatalogRegistry = exports.MikroOrmCatalogRegistry = exports.CATALOG_OVERLAY_STORE = exports.InMemoryCatalogOverlayStore = exports.FileCatalogOverlayStore = exports.CATALOG_OPTIONS = exports.isQueryStore = exports.assertReadOnlyShape = exports.CatalogModule = exports.emitCatalog = exports.curationActor = exports.channelNameFor = exports.catalogEventPhase = exports.UNATTRIBUTED_PRINCIPAL_ID = exports.CATALOG_LIB = exports.CATALOG_EVENTS = exports.CATALOG_EVENT_PHASE_FALLBACK = exports.CATALOG_EVENT_PHASE = exports.CatalogType = exports.CatalogProperty = void 0;
18
- exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.supportsCarryForward = exports.isWriteStore = exports.isReservedColumn = exports.isCatalogStoreCapabilities = exports.findColumnCollisions = exports.CatalogColumnCollisionError = exports.CATALOG_STORE = exports.CATALOG_SNAPSHOT_MODES = exports.CATALOG_RESERVED_COLUMNS = exports.assertNoColumnCollisions = exports.StaticKeyPrincipalResolver = exports.readableObjectPage = exports.mayWrite = exports.mayRead = exports.maySeeClassification = exports.PRINCIPAL_ACTOR_SEPARATOR = exports.parsePrincipalId = exports.hasScope = exports.expandScopes = exports.delegatePrincipal = exports.composePrincipalId = exports.CATALOG_PRINCIPAL_RESOLVER = exports.traceOutcomeFilter = exports.isWorkspaceStore = exports.isTraceStore = exports.isCatalogTraceOutcome = exports.embeddedVisualization = exports.CATALOG_WORKSPACE_STORE = exports.CATALOG_TRACE_STORE = exports.CATALOG_TRACE_OUTCOMES = exports.visibleToPrincipal = void 0;
17
+ exports.bestMatch = exports.MAX_SEARCH_LIMIT = exports.DEFAULT_SEARCH_LIMIT = exports.CatalogService = exports.SubprocessTransformRunner = exports.toCsv = exports.QueryCache = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.TRANSFORM_LANGUAGES = exports.TRANSFORM_RUNNER = exports.supportsWorkflowStages = exports.supportsWorkflows = exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowNode = exports.isWorkflowExecutionMode = exports.isWorkflowEdge = exports.supportsTransformRevisions = exports.isTransformLanguage = exports.isPipelineStore = exports.isConnectorKind = exports.CONNECTOR_KINDS = exports.CATALOG_PIPELINE_STORE = exports.CatalogRegistry = exports.MikroOrmCatalogRegistry = exports.CATALOG_OVERLAY_STORE = exports.InMemoryCatalogOverlayStore = exports.FileCatalogOverlayStore = exports.CATALOG_OPTIONS = exports.isQueryStore = exports.assertReadOnlyShape = exports.CatalogModule = exports.emitCatalog = exports.curationActor = exports.channelNameFor = exports.catalogEventPhase = exports.UNATTRIBUTED_PRINCIPAL_ID = exports.CATALOG_LIB = exports.CATALOG_EVENTS = exports.CATALOG_EVENT_PHASE_FALLBACK = exports.CATALOG_EVENT_PHASE = exports.CatalogType = exports.CatalogProperty = void 0;
18
+ exports.RequireScopes = exports.RequireHuman = exports.REQUIRES_HUMAN = exports.REQUIRED_SCOPES = exports.MikroOrmReadStore = exports.supportsCarryForward = exports.isWriteStore = exports.isReservedColumn = exports.isCatalogStoreCapabilities = exports.findColumnCollisions = exports.CatalogColumnCollisionError = exports.CATALOG_STORE = exports.CATALOG_SNAPSHOT_MODES = exports.CATALOG_RESERVED_COLUMNS = exports.assertNoColumnCollisions = exports.StaticKeyPrincipalResolver = exports.readableObjectPage = exports.mayWrite = exports.mayRead = exports.maySeeClassification = exports.PRINCIPAL_ACTOR_SEPARATOR = exports.parsePrincipalId = exports.hasScope = exports.expandScopes = exports.delegatePrincipal = exports.composePrincipalId = exports.CATALOG_PRINCIPAL_RESOLVER = exports.traceOutcomeFilter = exports.supportsSavedQueryRevisions = exports.isWorkspaceStore = exports.isTraceStore = exports.isCatalogTraceOutcome = exports.embeddedVisualization = exports.CATALOG_WORKSPACE_STORE = exports.CATALOG_TRACE_STORE = exports.CATALOG_TRACE_OUTCOMES = exports.CATALOG_REVISION_LIMIT = exports.visibleToPrincipal = exports.searchCatalog = exports.maySearch = exports.emptySearch = void 0;
19
19
  var catalog_decorators_1 = require("./catalog.decorators");
20
20
  Object.defineProperty(exports, "CatalogProperty", { enumerable: true, get: function () { return catalog_decorators_1.CatalogProperty; } });
21
21
  Object.defineProperty(exports, "CatalogType", { enumerable: true, get: function () { return catalog_decorators_1.CatalogType; } });
@@ -36,6 +36,14 @@ Object.defineProperty(exports, "assertReadOnlyShape", { enumerable: true, get: f
36
36
  Object.defineProperty(exports, "isQueryStore", { enumerable: true, get: function () { return catalog_query_1.isQueryStore; } });
37
37
  var catalog_options_1 = require("./catalog.options");
38
38
  Object.defineProperty(exports, "CATALOG_OPTIONS", { enumerable: true, get: function () { return catalog_options_1.CATALOG_OPTIONS; } });
39
+ // Everything, deliberately, and here more than anywhere: this is a seam two
40
+ // separate provider packages are being written against. A barrel that shipped
41
+ // `CATALOG_SECRET_VAULT` and `CatalogSecretVault` but not `SealedSecret` or
42
+ // `SecretContext` — the return type and the argument of the two methods a
43
+ // provider implements — would be the exact gap `index.barrel.spec.ts` was
44
+ // written after, reproduced on the one surface where a third party compiles
45
+ // against it.
46
+ __exportStar(require("./catalog.secrets"), exports);
39
47
  var catalog_overlay_store_1 = require("./catalog.overlay-store");
40
48
  Object.defineProperty(exports, "FileCatalogOverlayStore", { enumerable: true, get: function () { return catalog_overlay_store_1.FileCatalogOverlayStore; } });
41
49
  Object.defineProperty(exports, "InMemoryCatalogOverlayStore", { enumerable: true, get: function () { return catalog_overlay_store_1.InMemoryCatalogOverlayStore; } });
@@ -51,10 +59,12 @@ Object.defineProperty(exports, "CONNECTOR_KINDS", { enumerable: true, get: funct
51
59
  Object.defineProperty(exports, "isConnectorKind", { enumerable: true, get: function () { return catalog_pipeline_1.isConnectorKind; } });
52
60
  Object.defineProperty(exports, "isPipelineStore", { enumerable: true, get: function () { return catalog_pipeline_1.isPipelineStore; } });
53
61
  Object.defineProperty(exports, "isTransformLanguage", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformLanguage; } });
62
+ Object.defineProperty(exports, "supportsTransformRevisions", { enumerable: true, get: function () { return catalog_pipeline_1.supportsTransformRevisions; } });
54
63
  Object.defineProperty(exports, "isWorkflowEdge", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowEdge; } });
55
64
  Object.defineProperty(exports, "isWorkflowExecutionMode", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowExecutionMode; } });
56
65
  Object.defineProperty(exports, "isWorkflowNode", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowNode; } });
57
66
  Object.defineProperty(exports, "isWorkflowNodeKind", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowNodeKind; } });
67
+ Object.defineProperty(exports, "isWorkflowStatus", { enumerable: true, get: function () { return catalog_pipeline_1.isWorkflowStatus; } });
58
68
  Object.defineProperty(exports, "supportsWorkflows", { enumerable: true, get: function () { return catalog_pipeline_1.supportsWorkflows; } });
59
69
  Object.defineProperty(exports, "supportsWorkflowStages", { enumerable: true, get: function () { return catalog_pipeline_1.supportsWorkflowStages; } });
60
70
  Object.defineProperty(exports, "TRANSFORM_RUNNER", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_RUNNER; } });
@@ -64,6 +74,7 @@ Object.defineProperty(exports, "WORKFLOW_EXECUTION_MODES", { enumerable: true, g
64
74
  Object.defineProperty(exports, "WORKFLOW_ISSUE_CODES", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_ISSUE_CODES; } });
65
75
  Object.defineProperty(exports, "WORKFLOW_NODE_ID_PATTERN", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_NODE_ID_PATTERN; } });
66
76
  Object.defineProperty(exports, "WORKFLOW_NODE_KINDS", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_NODE_KINDS; } });
77
+ Object.defineProperty(exports, "WORKFLOW_STATUSES", { enumerable: true, get: function () { return catalog_pipeline_1.WORKFLOW_STATUSES; } });
67
78
  Object.defineProperty(exports, "workflowGraphHash", { enumerable: true, get: function () { return catalog_pipeline_1.workflowGraphHash; } });
68
79
  Object.defineProperty(exports, "workflowRunOrder", { enumerable: true, get: function () { return catalog_pipeline_1.workflowRunOrder; } });
69
80
  // The environment surface: which catalog database a call is served from, and
@@ -92,6 +103,7 @@ Object.defineProperty(exports, "maySearch", { enumerable: true, get: function ()
92
103
  Object.defineProperty(exports, "searchCatalog", { enumerable: true, get: function () { return search_1.searchCatalog; } });
93
104
  Object.defineProperty(exports, "visibleToPrincipal", { enumerable: true, get: function () { return search_1.visibleToPrincipal; } });
94
105
  var catalog_workspace_1 = require("./catalog.workspace");
106
+ Object.defineProperty(exports, "CATALOG_REVISION_LIMIT", { enumerable: true, get: function () { return catalog_workspace_1.CATALOG_REVISION_LIMIT; } });
95
107
  Object.defineProperty(exports, "CATALOG_TRACE_OUTCOMES", { enumerable: true, get: function () { return catalog_workspace_1.CATALOG_TRACE_OUTCOMES; } });
96
108
  Object.defineProperty(exports, "CATALOG_TRACE_STORE", { enumerable: true, get: function () { return catalog_workspace_1.CATALOG_TRACE_STORE; } });
97
109
  Object.defineProperty(exports, "CATALOG_WORKSPACE_STORE", { enumerable: true, get: function () { return catalog_workspace_1.CATALOG_WORKSPACE_STORE; } });
@@ -99,6 +111,7 @@ Object.defineProperty(exports, "embeddedVisualization", { enumerable: true, get:
99
111
  Object.defineProperty(exports, "isCatalogTraceOutcome", { enumerable: true, get: function () { return catalog_workspace_1.isCatalogTraceOutcome; } });
100
112
  Object.defineProperty(exports, "isTraceStore", { enumerable: true, get: function () { return catalog_workspace_1.isTraceStore; } });
101
113
  Object.defineProperty(exports, "isWorkspaceStore", { enumerable: true, get: function () { return catalog_workspace_1.isWorkspaceStore; } });
114
+ Object.defineProperty(exports, "supportsSavedQueryRevisions", { enumerable: true, get: function () { return catalog_workspace_1.supportsSavedQueryRevisions; } });
102
115
  Object.defineProperty(exports, "traceOutcomeFilter", { enumerable: true, get: function () { return catalog_workspace_1.traceOutcomeFilter; } });
103
116
  var catalog_principal_1 = require("./catalog.principal");
104
117
  Object.defineProperty(exports, "CATALOG_PRINCIPAL_RESOLVER", { enumerable: true, get: function () { return catalog_principal_1.CATALOG_PRINCIPAL_RESOLVER; } });
package/dist/transform-runner.d.ts CHANGED
@@ -15,11 +15,28 @@ export interface TransformRunnerOptions {
15
15
  /**
16
16
  * Runs a transform in a child process, with a clock on it.
17
17
  *
18
- * **This is not a security boundary.** It stops accidents — an infinite loop, a
19
- * runaway allocation, a stray read of `process.env.DATABASE_PASSWORD` — because
20
- * the child gets a timeout and an empty environment. It does not stop code
21
- * written to escape it: a child process can still open sockets and read the
22
- * filesystem as whatever user the service runs as.
18
+ * **This is not a security boundary, and the trimmed environment is not one
19
+ * either.** It stops accidents — an infinite loop, a runaway allocation, a stray
20
+ * read of `process.env.DATABASE_PASSWORD` — because the child gets a timeout and
21
+ * an environment of `{PATH, NODE_ENV}`. It does not stop code written to escape
22
+ * it, and it is worth being exact about how thin the allowlist is rather than
23
+ * leaving a reader to assume it holds:
24
+ *
25
+ * - the child inherits nothing of the parent's environment **through `env`**,
26
+ * and reads all of it anyway from `/proc/<ppid>/environ`, which is readable
27
+ * because parent and child run as the same uid;
28
+ * - it runs in a working directory of this runner's choosing but on the host's
29
+ * filesystem, so a service account token under
30
+ * `/var/run/secrets/kubernetes.io/serviceaccount/` is an absolute path away;
31
+ * - it can open sockets, as whatever user the service runs as.
32
+ *
33
+ * So the allowlist is a guard rail against the accident, and the reachability of
34
+ * everything it names is a property of the process boundary, not a leak to be
35
+ * patched. **Running a transform is running code in this pod.** Who is allowed
36
+ * to is therefore an authorisation question and not a sandboxing one, and it is
37
+ * answered at the HTTP surface — see the "Running a transform is running code"
38
+ * section of `@dudousxd/nestjs-catalog-pipeline`'s README, which is where a host
39
+ * can actually read it, and `pipeline.controller.ts` for the checks themselves.
23
40
  *
24
41
  * That is a deliberate trade for the case this is built for, where transforms
25
42
  * are written by the same people who already have database access. A catalog
package/dist/transform-runner.js CHANGED
@@ -13,10 +13,50 @@ Object.defineProperty(exports, "__esModule", { value: true });
13
13
  exports.SubprocessTransformRunner = void 0;
14
14
  const node_child_process_1 = require("node:child_process");
15
15
  const node_fs_1 = require("node:fs");
16
+ const node_os_1 = require("node:os");
16
17
  const node_path_1 = require("node:path");
17
18
  const common_1 = require("@nestjs/common");
18
19
  const DEFAULT_TIMEOUT_MS = 30_000;
19
20
  const MAX_OUTPUT_BYTES = 32 * 1024 * 1024;
21
+ /**
22
+ * How much of the child's stderr is held, and why it is a different number from
23
+ * {@link MAX_OUTPUT_BYTES} with a different consequence.
24
+ *
25
+ * It had no bound at all, which is the one shape a capture must never have when
26
+ * the thing filling it is user code: `stderr += chunk` ran for the whole timeout
27
+ * window, so a transform whose only line is a loop writing to fd 2 grew the
28
+ * **parent's** heap — not the child's — at whatever rate the pipe would carry,
29
+ * and took the pod out with it. The timeout is no answer to that: thirty seconds
30
+ * of an unthrottled writer is gigabytes, and the process that dies is the one
31
+ * serving every other request.
32
+ *
33
+ * Bounded rather than killed, which is the opposite of what stdout overflow
34
+ * does, and the asymmetry is the point. Stdout *is* the result channel — past
35
+ * {@link MAX_OUTPUT_BYTES} there is no readable JSON line at the end of it and
36
+ * the run has already failed, so killing costs nothing. Stderr is only ever the
37
+ * diagnostic: a transform that writes a great deal to it and then returns a
38
+ * perfectly good array of rows is a working transform, and killing it would turn
39
+ * a noisy dependency's warnings into a failed load.
40
+ *
41
+ * The **head** is kept, because the head is what is read. Both places that
42
+ * consume this take `stderr.slice(0, 500)` — the first line of a traceback, the
43
+ * import error, the thing that says what went wrong — so dropping the tail
44
+ * discards exactly the part nobody was going to see. 64 KiB is far more than any
45
+ * of those and small enough that the ceiling is not itself a memory decision.
46
+ */
47
+ const MAX_CAPTURED_STDERR_BYTES = 64 * 1024;
48
+ /**
49
+ * Whether the child is put in its own process group, so that stopping it stops
50
+ * what it started.
51
+ *
52
+ * POSIX only, because the mechanism is POSIX: `detached` there makes the child a
53
+ * process-group leader and `process.kill(-pid)` signals the whole group, which
54
+ * is the only way to reach a grandchild. On Windows `detached` means something
55
+ * else entirely (a new console) and a negative pid is not a group, so the
56
+ * platform gets the single-process kill it always had rather than a call that
57
+ * would throw on every timeout.
58
+ */
59
+ const KILL_PROCESS_GROUP = process.platform !== 'win32';
20
60
  /**
21
61
  * How much of what a transform logged is carried back, on both axes.
22
62
  *
@@ -73,11 +113,28 @@ const REPORTED_PACKAGES = ['pandas', 'numpy', 'pyarrow', 'requests'];
73
113
  /**
74
114
  * Runs a transform in a child process, with a clock on it.
75
115
  *
76
- * **This is not a security boundary.** It stops accidents — an infinite loop, a
77
- * runaway allocation, a stray read of `process.env.DATABASE_PASSWORD` — because
78
- * the child gets a timeout and an empty environment. It does not stop code
79
- * written to escape it: a child process can still open sockets and read the
80
- * filesystem as whatever user the service runs as.
116
+ * **This is not a security boundary, and the trimmed environment is not one
117
+ * either.** It stops accidents — an infinite loop, a runaway allocation, a stray
118
+ * read of `process.env.DATABASE_PASSWORD` — because the child gets a timeout and
119
+ * an environment of `{PATH, NODE_ENV}`. It does not stop code written to escape
120
+ * it, and it is worth being exact about how thin the allowlist is rather than
121
+ * leaving a reader to assume it holds:
122
+ *
123
+ * - the child inherits nothing of the parent's environment **through `env`**,
124
+ * and reads all of it anyway from `/proc/<ppid>/environ`, which is readable
125
+ * because parent and child run as the same uid;
126
+ * - it runs in a working directory of this runner's choosing but on the host's
127
+ * filesystem, so a service account token under
128
+ * `/var/run/secrets/kubernetes.io/serviceaccount/` is an absolute path away;
129
+ * - it can open sockets, as whatever user the service runs as.
130
+ *
131
+ * So the allowlist is a guard rail against the accident, and the reachability of
132
+ * everything it names is a property of the process boundary, not a leak to be
133
+ * patched. **Running a transform is running code in this pod.** Who is allowed
134
+ * to is therefore an authorisation question and not a sandboxing one, and it is
135
+ * answered at the HTTP surface — see the "Running a transform is running code"
136
+ * section of `@dudousxd/nestjs-catalog-pipeline`'s README, which is where a host
137
+ * can actually read it, and `pipeline.controller.ts` for the checks themselves.
81
138
  *
82
139
  * That is a deliberate trade for the case this is built for, where transforms
83
140
  * are written by the same people who already have database access. A catalog
@@ -179,7 +236,21 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
179
236
  const child = (0, node_child_process_1.spawn)(command, args, {
180
237
  // An empty environment, not the parent's. A transform has no business
181
238
  // reading the database password, and inheriting env is how it would.
239
+ // Read the class docblock before treating this as containment: the same
240
+ // values are a `/proc/<ppid>/environ` read away, and this is a guard
241
+ // rail against the accidental read rather than a boundary.
182
242
  env: { PATH: process.env.PATH ?? '', NODE_ENV: 'production' },
243
+ // Not the parent's, which is a running service's directory and holds
244
+ // the `.env` the allowlist above exists to withhold — a transform whose
245
+ // first line is `readFileSync(".env")` was reading the host application's
246
+ // configuration by relative path. A temporary directory keeps the file
247
+ // writes a transform may legitimately want working while making the one
248
+ // path it can name without knowing anything about the deployment
249
+ // uninteresting. Absolute paths are unaffected, and cannot be.
250
+ cwd: (0, node_os_1.tmpdir)(),
251
+ // Its own process group, so the timeout below can reach a grandchild.
252
+ // See {@link KILL_PROCESS_GROUP}.
253
+ detached: KILL_PROCESS_GROUP,
183
254
  stdio: ['pipe', 'pipe', 'pipe'],
184
255
  });
185
256
  let stdout = '';
@@ -189,16 +260,22 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
189
260
  if (settled)
190
261
  return;
191
262
  settled = true;
192
- child.kill('SIGKILL');
263
+ stop(child);
193
264
  reject(new Error(`The transform ran for longer than ${timeoutMs}ms and was stopped.`));
194
265
  }, timeoutMs);
195
266
  child.stdout.on('data', (chunk) => {
196
267
  stdout += chunk.toString();
197
268
  if (stdout.length > MAX_OUTPUT_BYTES)
198
- child.kill('SIGKILL');
269
+ stop(child);
199
270
  });
200
271
  child.stderr.on('data', (chunk) => {
201
- stderr += chunk.toString();
272
+ // Appended only while there is room, rather than appended and trimmed:
273
+ // trimming after the fact still materialises the whole chunk into the
274
+ // parent's heap, which is the thing being bounded. See
275
+ // {@link MAX_CAPTURED_STDERR_BYTES} for why this bounds rather than kills.
276
+ if (stderr.length >= MAX_CAPTURED_STDERR_BYTES)
277
+ return;
278
+ stderr += chunk.toString().slice(0, MAX_CAPTURED_STDERR_BYTES - stderr.length);
202
279
  });
203
280
  child.on('error', (error) => {
204
281
  if (settled)
@@ -228,7 +305,12 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
228
305
  return this.pythonPath;
229
306
  const venv = this.options.pythonVenv ?? process.env.CATALOG_PYTHON_VENV;
230
307
  if (venv) {
231
- const candidate = (0, node_path_1.join)(venv, 'bin', 'python');
308
+ // Absolute, and it has to be: a child now runs in a temporary directory
309
+ // rather than the parent's, so a relative `CATALOG_PYTHON_VENV` — which
310
+ // `existsSync` here resolves against the *parent's* cwd — would pass this
311
+ // check and then fail to spawn. Resolved once, at the point the two cwds
312
+ // are still the same.
313
+ const candidate = (0, node_path_1.resolve)((0, node_path_1.join)(venv, 'bin', 'python'));
232
314
  if ((0, node_fs_1.existsSync)(candidate)) {
233
315
  this.pythonPath = candidate;
234
316
  this.logger.log(`Python transforms run in the venv at ${venv}`);
@@ -256,6 +338,41 @@ exports.SubprocessTransformRunner = SubprocessTransformRunner = SubprocessTransf
256
338
  (0, common_1.Injectable)(),
257
339
  __metadata("design:paramtypes", [Object])
258
340
  ], SubprocessTransformRunner);
341
+ /**
342
+ * Stop the transform, and everything the transform started.
343
+ *
344
+ * `child.kill()` signals one pid. A transform that double-forks — two lines of
345
+ * `child_process.spawn` with `detached` and an `unref` — leaves a grandchild
346
+ * that the direct child's death says nothing about, so the timeout expired, the
347
+ * caller was told the run had been stopped, and the work carried on
348
+ * indefinitely. That is not a hypothetical: it is the standard way a timeout on
349
+ * a process is escaped, and a bound that a caller can opt out of is not a bound.
350
+ *
351
+ * So the child leads its own process group and the negative pid signals the
352
+ * group, which is every descendant that has not deliberately left it. Leaving
353
+ * one is possible (`setsid` again) and there is no answer to that short of a
354
+ * cgroup or a container — the same place the class docblock's honesty about the
355
+ * boundary ends up, for the same reason.
356
+ *
357
+ * Falls through to the single-process kill whenever the group kill cannot be the
358
+ * one that happens: on Windows, where a negative pid is not a group, and on an
359
+ * `ESRCH` where the group is already gone and the direct kill is a harmless
360
+ * no-op. A throw here would replace a timeout error — which says something true
361
+ * and useful — with an unhandled one that says nothing.
362
+ */
363
+ function stop(child) {
364
+ const pid = child.pid;
365
+ if (KILL_PROCESS_GROUP && pid !== undefined) {
366
+ try {
367
+ process.kill(-pid, 'SIGKILL');
368
+ return;
369
+ }
370
+ catch {
371
+ // Already gone, or never grouped. The direct kill below covers both.
372
+ }
373
+ }
374
+ child.kill('SIGKILL');
375
+ }
259
376
  /**
260
377
  * The traceback, plus the tail of what the code printed on its way to it.
261
378
  *
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dudousxd/nestjs-catalog",
3
- "version": "0.10.0",
3
+ "version": "0.12.0",
4
4
  "description": "A metadata registry for NestJS: object types, properties and relations, derived from your ORM and enriched with decorators.",
5
5
  "license": "MIT",
6
6
  "author": "Davide Carvalho",