@burdenoff/be-sdk 2026.1005.1 → 2026.1006.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -0
- package/dist/index.js +2 -2
- package/dist/lake/index.d.ts +59 -0
- package/dist/lake/index.js +253 -0
- package/dist/lake/index.js.map +1 -0
- package/package.json +6 -3
package/README.md
CHANGED
|
@@ -204,3 +204,67 @@ PROPRIETARY - Copyright Burdenoff Consultancy Services Pvt. Ltd.
|
|
|
204
204
|
## Support
|
|
205
205
|
|
|
206
206
|
For support, email contact@burdenoff.com or visit [https://burdenoff.com/support](https://burdenoff.com/support).
|
|
207
|
+
|
|
208
|
+
## Shared DuckLake dataset contract (BOFF-8254)
|
|
209
|
+
|
|
210
|
+
The DuckLake catalog PostgreSQL database owns `shared_lake_datasets` and
|
|
211
|
+
`shared_lake_dataset_consumers` (migration in `prisma/lake-registry/`).
|
|
212
|
+
`dataset_id` is the existing BigConsole sink UUID, `metadata_schema` is
|
|
213
|
+
`dl_<UUID with underscores>`, and the S3 prefix stays
|
|
214
|
+
`datasinks/<dataset_id>/`. BigConsole's service database continues to own the
|
|
215
|
+
ingest watermark and dashboard payload behavior; it mirrors legacy metadata
|
|
216
|
+
to this registry. Run its idempotent `scripts/backfill-shared-lake-registry.ts`
|
|
217
|
+
after the catalog migration, then let normal writes keep it current.
|
|
218
|
+
|
|
219
|
+
Legacy `payload_versions` and `payload_elements` are JSON dashboard data, not
|
|
220
|
+
an operational TimeCampus contract. A product owner adds a typed table in the
|
|
221
|
+
same catalog schema (or a new dataset), with non-null `workspace_id` and
|
|
222
|
+
nullable `product_id`. A null `product_id` explicitly marks a row shared with
|
|
223
|
+
all **approved** dataset consumers. Product-specific rows carry their product
|
|
224
|
+
slug. Readers use both `workspace_id` and `(product_id = consumer OR
|
|
225
|
+
product_id IS NULL)` in every query. Column names and types are versioned by
|
|
226
|
+
`schema_version`; an incompatible change requires a new version/table and a
|
|
227
|
+
staged consumer migration. A single designated owner writes each typed table;
|
|
228
|
+
readers never write. Owners serialize commits and update
|
|
229
|
+
`last_committed_version` only after the lake transaction succeeds. Add DuckLake
|
|
230
|
+
product/date partitions only after file-size and query-profile evidence shows
|
|
231
|
+
that pruning is useful; partitions are a performance choice, not an access
|
|
232
|
+
control mechanism.
|
|
233
|
+
|
|
234
|
+
The first TimeCampus GraphQL operation exposes only typed `time_entries`
|
|
235
|
+
tables classified L1 or L2. Other approved datasets remain service-only until
|
|
236
|
+
their fields and end-user RBAC are reviewed explicitly.
|
|
237
|
+
|
|
238
|
+
Rollout for a TimeCampus grant:
|
|
239
|
+
|
|
240
|
+
1. Apply `bun run lake:registry:migrate` against the catalog with
|
|
241
|
+
`DUCKLAKE_REGISTRY_MIGRATION_URL`. Backfill existing BigConsole sinks.
|
|
242
|
+
2. Create the typed table and a representative fixture in alpha, validate
|
|
243
|
+
workspace/product columns and version, then call `registerTyped` with the
|
|
244
|
+
original sink ID/routing and `lifecycle: 'ready'`. BigConsole continues using
|
|
245
|
+
its legacy payload tables in the same schema.
|
|
246
|
+
3. Add the dataset UUID to `timecampus_ducklake_dataset_ids.alpha` in
|
|
247
|
+
`tf-aws-cloud/roots/secrets`, review/apply the S3 policy through IaC, and
|
|
248
|
+
provision a separate static access key in the TimeCampus Secrets Manager
|
|
249
|
+
bundle. The TimeCampus catalog login must be a member of the
|
|
250
|
+
`boff_timecampus_lake_reader` role; put its read-only URL in
|
|
251
|
+
`DUCKLAKE_CATALOG_URL`. Add `DUCKLAKE_S3_BUCKET`,
|
|
252
|
+
`DUCKLAKE_S3_ACCESS_KEY_ID`, and `DUCKLAKE_S3_SECRET_ACCESS_KEY` to the same
|
|
253
|
+
bundle. The workload already imports its bundle through GitOps.
|
|
254
|
+
4. Run `bun run lake:grant:timecampus` with the approved `LAKE_DATASET_ID` and
|
|
255
|
+
`LAKE_GRANT_ACTOR`. This grants catalog schema SELECT and activates the
|
|
256
|
+
product grant. The approved view prevents TimeCampus's catalog login from
|
|
257
|
+
reading the unfiltered registry tables. The IAM key can read only the
|
|
258
|
+
reviewed S3 prefix. Validate discovery, direct read, denied guessed ID, and
|
|
259
|
+
mixed-product rows with
|
|
260
|
+
`timecampus-doctor/wspace/modules/timecampus/scripts/test-timecampus.sh --test lake`
|
|
261
|
+
in alpha; then
|
|
262
|
+
repeat the reviewed grant and validation in prod.
|
|
263
|
+
|
|
264
|
+
To adopt a second product, add its own catalog view and NOLOGIN reader role,
|
|
265
|
+
per-dataset PostgreSQL schema grants, IAM principal/prefix policy, Secrets
|
|
266
|
+
Manager bundle values, and service using `LakeDatasetRegistry` plus
|
|
267
|
+
`AuthorizedLakeReader`. Grant revocation must use
|
|
268
|
+
`LAKE_GRANT_ACTION=revoke bun run lake:grant:timecampus`, remove the dataset
|
|
269
|
+
UUID from the Terraform S3 policy, and rotate credentials if a removed product
|
|
270
|
+
previously held them. Do not share BigConsole's broad lake credentials.
|
package/dist/index.js
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
|
+
export { sharing_exports as Sharing } from './chunk-DS3YMJ4X.js';
|
|
1
2
|
export { TemporalClient, TemporalClientFactory, TemporalModule, TemporalWorker, TemporalWorkerFactory, TemporalWorkerManager } from './chunk-YNDJQRI3.js';
|
|
2
3
|
import { ActivityModule, UsageModule, TrackerModule } from './chunk-NCC5UN3L.js';
|
|
3
4
|
export { ActivityModule, TrackerModule, UsageModule } from './chunk-NCC5UN3L.js';
|
|
4
5
|
export { NAME_MAX_LENGTH, NAME_MIN_LENGTH, USERNAME_MAX_LENGTH, USERNAME_MIN_LENGTH, validation_exports as Validation, lengthInCodePoints, validateName, validateUserFields, validateUsername } from './chunk-AHEXIONU.js';
|
|
6
|
+
export { observability_exports as Observability } from './chunk-VAT66FJA.js';
|
|
5
7
|
export { platform_exports as Platform } from './chunk-Z5BURBFZ.js';
|
|
6
8
|
export { prisma_exports as Prisma } from './chunk-CVMQRWCB.js';
|
|
7
9
|
export { PRODUCT_RENAMES, PRODUCT_RENAME_EMIT, PRODUCT_SLUG_ALIASES, product_alias_exports as ProductAlias, RENAME_CONCERNS, billingIdVariants, emitBillingId, emitName, emitProductSlug, emitQuotaKey, emitRbacResource, emitTemplateKey, isLegacyName, isSameBillingId, isSameGraphQLName, isSameName, isSameProductSlug, isSameQuotaKey, isSameRbacResource, isSameTemplateKey, nameVariants, normalizeBillingId, normalizeGraphQLName, normalizeProductSlug, normalizeQuotaKey, normalizeRbacResource, normalizeTemplateKey, productSlugVariants, quotaKeyVariants, rbacResourceVariants, readRenamedEnv, templateKeyVariants, toCurrentName, toLegacyName } from './chunk-OBBCR6WY.js';
|
|
8
10
|
import { RBACModule } from './chunk-5JNHVKHG.js';
|
|
9
11
|
export { RBACModule, createRBACDirective, createRBACDirectiveForLayer, createRBACDirectiveFromEnv } from './chunk-5JNHVKHG.js';
|
|
10
12
|
import './chunk-FA5ROGTS.js';
|
|
11
|
-
export { sharing_exports as Sharing } from './chunk-DS3YMJ4X.js';
|
|
12
13
|
export { instrumentation_exports as Instrumentation } from './chunk-E5DTAGCA.js';
|
|
13
14
|
export { middleware_exports as Middleware } from './chunk-ELU7VL3E.js';
|
|
14
15
|
import { TenantNotificationsModule, NotificationsModule } from './chunk-INAZ3PKF.js';
|
|
15
16
|
export { tenant_notifications_exports as Notifications, NotificationsModule } from './chunk-INAZ3PKF.js';
|
|
16
|
-
export { observability_exports as Observability } from './chunk-VAT66FJA.js';
|
|
17
17
|
export { export_exports as Export } from './chunk-RFXAHMY4.js';
|
|
18
18
|
export { feature_flags_exports as FeatureFlags } from './chunk-UNASM5VJ.js';
|
|
19
19
|
import { GraphModule } from './chunk-VVBA3LQV.js';
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { Pool } from 'pg';
|
|
2
|
+
|
|
3
|
+
type DatasetLifecycle = 'provisioning' | 'ready' | 'archived';
|
|
4
|
+
type DatasetRepresentation = 'legacy_payload' | 'typed_table';
|
|
5
|
+
type DatasetClassification = 'L1_PUBLIC' | 'L2_INTERNAL' | 'L3_CONFIDENTIAL' | 'L4_REGULATED';
|
|
6
|
+
interface LakeDataset {
|
|
7
|
+
datasetId: string;
|
|
8
|
+
workspaceId: string;
|
|
9
|
+
datasetKey: string;
|
|
10
|
+
ownerProduct: string;
|
|
11
|
+
sourceProduct: string;
|
|
12
|
+
metadataSchema: string;
|
|
13
|
+
dataPath: string;
|
|
14
|
+
representation: DatasetRepresentation;
|
|
15
|
+
typedTable: string | null;
|
|
16
|
+
rowProductColumn: string | null;
|
|
17
|
+
schemaVersion: number;
|
|
18
|
+
classification: DatasetClassification;
|
|
19
|
+
lifecycle: DatasetLifecycle;
|
|
20
|
+
lastCommittedVersion: number;
|
|
21
|
+
}
|
|
22
|
+
interface RegisterLakeDatasetInput extends Omit<LakeDataset, 'lifecycle' | 'lastCommittedVersion'> {
|
|
23
|
+
lifecycle?: DatasetLifecycle;
|
|
24
|
+
lastCommittedVersion?: number;
|
|
25
|
+
}
|
|
26
|
+
declare function validateLakeDatasetIdentity(dataset: Pick<LakeDataset, 'datasetId' | 'workspaceId' | 'datasetKey' | 'ownerProduct' | 'sourceProduct' | 'metadataSchema' | 'dataPath' | 'typedTable' | 'rowProductColumn' | 'representation'>): void;
|
|
27
|
+
declare class LakeDatasetRegistry {
|
|
28
|
+
private readonly db;
|
|
29
|
+
private readonly approvedView;
|
|
30
|
+
constructor(db: Pool, approvedView?: 'shared_lake_timecampus_approved' | null);
|
|
31
|
+
private approvedSource;
|
|
32
|
+
private approvalPredicate;
|
|
33
|
+
private approvalCheck;
|
|
34
|
+
getApprovedById(workspaceId: string, productId: string, datasetId: string): Promise<LakeDataset | null>;
|
|
35
|
+
getApprovedByKey(workspaceId: string, productId: string, datasetKey: string): Promise<LakeDataset | null>;
|
|
36
|
+
listApproved(workspaceId: string, productId: string, limit?: number): Promise<LakeDataset[]>;
|
|
37
|
+
registerLegacy(input: RegisterLakeDatasetInput, actorId: string): Promise<void>;
|
|
38
|
+
registerTyped(input: RegisterLakeDatasetInput, actorId: string): Promise<void>;
|
|
39
|
+
grantConsumer(datasetId: string, productId: string, actorId: string): Promise<void>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
interface LakeReadExecutor {
|
|
43
|
+
queryReadOnly(dataset: LakeDataset, sql: string, parameters: readonly (string | number)[]): Promise<Record<string, unknown>[]>;
|
|
44
|
+
}
|
|
45
|
+
interface LakeReadPage {
|
|
46
|
+
datasetId: string;
|
|
47
|
+
schemaVersion: number;
|
|
48
|
+
lastCommittedVersion: number;
|
|
49
|
+
rows: Record<string, unknown>[];
|
|
50
|
+
}
|
|
51
|
+
declare class AuthorizedLakeReader {
|
|
52
|
+
private readonly registry;
|
|
53
|
+
private readonly executor;
|
|
54
|
+
private readonly productId;
|
|
55
|
+
constructor(registry: LakeDatasetRegistry, executor: LakeReadExecutor, productId: string);
|
|
56
|
+
readRows(workspaceId: string, datasetId: string, limit?: number, offset?: number): Promise<LakeReadPage | null>;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export { AuthorizedLakeReader, type DatasetClassification, type DatasetLifecycle, type DatasetRepresentation, type LakeDataset, LakeDatasetRegistry, type LakeReadExecutor, type LakeReadPage, type RegisterLakeDatasetInput, validateLakeDatasetIdentity };
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
import { __name } from '../chunk-XE4OXN2W.js';
|
|
2
|
+
|
|
3
|
+
// src/lake/dataset-registry.ts
|
|
4
|
+
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
5
|
+
var PRODUCT_RE = /^[a-z][a-z0-9-]{0,63}$/;
|
|
6
|
+
var KEY_RE = /^[^\x00-\x1f\x7f]{1,100}$/;
|
|
7
|
+
var TABLE_RE = /^[a-z][a-z0-9_]{0,62}$/;
|
|
8
|
+
function validateLakeDatasetIdentity(dataset) {
|
|
9
|
+
if (!UUID_RE.test(dataset.datasetId) || !UUID_RE.test(dataset.workspaceId)) throw new Error("Invalid dataset identity");
|
|
10
|
+
if (!KEY_RE.test(dataset.datasetKey)) throw new Error("Invalid dataset key");
|
|
11
|
+
if (!PRODUCT_RE.test(dataset.ownerProduct) || !PRODUCT_RE.test(dataset.sourceProduct)) {
|
|
12
|
+
throw new Error("Invalid dataset product");
|
|
13
|
+
}
|
|
14
|
+
const expectedSchema = `dl_${dataset.datasetId.replaceAll("-", "_")}`;
|
|
15
|
+
if (dataset.metadataSchema !== expectedSchema) throw new Error("Dataset catalog schema does not match its ID");
|
|
16
|
+
const path = new URL(dataset.dataPath);
|
|
17
|
+
if (path.protocol !== "s3:" || !path.hostname || path.pathname !== `/datasinks/${dataset.datasetId}/`) {
|
|
18
|
+
throw new Error("Dataset S3 path does not match its ID");
|
|
19
|
+
}
|
|
20
|
+
if (dataset.representation === "typed_table") {
|
|
21
|
+
if (!dataset.typedTable || !TABLE_RE.test(dataset.typedTable)) throw new Error("Invalid typed table");
|
|
22
|
+
if (dataset.rowProductColumn !== "product_id") throw new Error("Typed shared tables must use product_id");
|
|
23
|
+
} else if (dataset.typedTable || dataset.rowProductColumn) {
|
|
24
|
+
throw new Error("Legacy payload datasets cannot declare a typed table");
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
__name(validateLakeDatasetIdentity, "validateLakeDatasetIdentity");
|
|
28
|
+
function toDataset(row) {
|
|
29
|
+
const dataset = {
|
|
30
|
+
datasetId: String(row["dataset_id"]),
|
|
31
|
+
workspaceId: String(row["workspace_id"]),
|
|
32
|
+
datasetKey: String(row["dataset_key"]),
|
|
33
|
+
ownerProduct: String(row["owner_product"]),
|
|
34
|
+
sourceProduct: String(row["source_product"]),
|
|
35
|
+
metadataSchema: String(row["metadata_schema"]),
|
|
36
|
+
dataPath: String(row["data_path"]),
|
|
37
|
+
representation: row["representation"],
|
|
38
|
+
typedTable: row["typed_table"] === null ? null : String(row["typed_table"]),
|
|
39
|
+
rowProductColumn: row["row_product_column"] === null ? null : String(row["row_product_column"]),
|
|
40
|
+
schemaVersion: Number(row["schema_version"]),
|
|
41
|
+
classification: row["classification"],
|
|
42
|
+
lifecycle: row["lifecycle"],
|
|
43
|
+
lastCommittedVersion: Number(row["last_committed_version"])
|
|
44
|
+
};
|
|
45
|
+
validateLakeDatasetIdentity(dataset);
|
|
46
|
+
return dataset;
|
|
47
|
+
}
|
|
48
|
+
__name(toDataset, "toDataset");
|
|
49
|
+
var LakeDatasetRegistry = class {
|
|
50
|
+
static {
|
|
51
|
+
__name(this, "LakeDatasetRegistry");
|
|
52
|
+
}
|
|
53
|
+
db;
|
|
54
|
+
approvedView;
|
|
55
|
+
constructor(db, approvedView = null) {
|
|
56
|
+
this.db = db;
|
|
57
|
+
this.approvedView = approvedView;
|
|
58
|
+
}
|
|
59
|
+
approvedSource() {
|
|
60
|
+
return this.approvedView ? `${this.approvedView} d` : "shared_lake_datasets d JOIN shared_lake_dataset_consumers c ON c.dataset_id = d.dataset_id";
|
|
61
|
+
}
|
|
62
|
+
approvalPredicate() {
|
|
63
|
+
return this.approvedView ? "d.product_id" : "c.product_id";
|
|
64
|
+
}
|
|
65
|
+
approvalCheck() {
|
|
66
|
+
return this.approvedView ? "" : "AND c.can_read = true";
|
|
67
|
+
}
|
|
68
|
+
async getApprovedById(workspaceId, productId, datasetId) {
|
|
69
|
+
if (!UUID_RE.test(workspaceId) || !UUID_RE.test(datasetId) || !PRODUCT_RE.test(productId)) return null;
|
|
70
|
+
const result = await this.db.query(`SELECT d.* FROM ${this.approvedSource()}
|
|
71
|
+
WHERE d.workspace_id = $1::uuid AND d.dataset_id = $2::uuid
|
|
72
|
+
AND ${this.approvalPredicate()} = $3 ${this.approvalCheck()} AND d.lifecycle = 'ready'`, [
|
|
73
|
+
workspaceId,
|
|
74
|
+
datasetId,
|
|
75
|
+
productId
|
|
76
|
+
]);
|
|
77
|
+
return result.rows[0] ? toDataset(result.rows[0]) : null;
|
|
78
|
+
}
|
|
79
|
+
async getApprovedByKey(workspaceId, productId, datasetKey) {
|
|
80
|
+
if (!UUID_RE.test(workspaceId) || !PRODUCT_RE.test(productId) || !KEY_RE.test(datasetKey)) return null;
|
|
81
|
+
const result = await this.db.query(`SELECT d.* FROM ${this.approvedSource()}
|
|
82
|
+
WHERE d.workspace_id = $1::uuid AND d.dataset_key = $2
|
|
83
|
+
AND ${this.approvalPredicate()} = $3 ${this.approvalCheck()} AND d.lifecycle = 'ready'`, [
|
|
84
|
+
workspaceId,
|
|
85
|
+
datasetKey,
|
|
86
|
+
productId
|
|
87
|
+
]);
|
|
88
|
+
return result.rows[0] ? toDataset(result.rows[0]) : null;
|
|
89
|
+
}
|
|
90
|
+
async listApproved(workspaceId, productId, limit = 100) {
|
|
91
|
+
if (!UUID_RE.test(workspaceId) || !PRODUCT_RE.test(productId)) return [];
|
|
92
|
+
const result = await this.db.query(`SELECT d.* FROM ${this.approvedSource()}
|
|
93
|
+
WHERE d.workspace_id = $1::uuid AND ${this.approvalPredicate()} = $2
|
|
94
|
+
${this.approvalCheck()} AND d.lifecycle = 'ready'
|
|
95
|
+
ORDER BY d.dataset_key ASC LIMIT $3`, [
|
|
96
|
+
workspaceId,
|
|
97
|
+
productId,
|
|
98
|
+
Math.min(Math.max(limit, 1), 1e3)
|
|
99
|
+
]);
|
|
100
|
+
return result.rows.map((row) => toDataset(row));
|
|
101
|
+
}
|
|
102
|
+
/** BigConsole's legacy sink registration: stable datasetId=sinkId and same Parquet prefix. */
|
|
103
|
+
async registerLegacy(input, actorId) {
|
|
104
|
+
validateLakeDatasetIdentity(input);
|
|
105
|
+
if (input.representation !== "legacy_payload" || input.ownerProduct !== "bigconsole") {
|
|
106
|
+
throw new Error("Only BigConsole legacy sinks may use registerLegacy");
|
|
107
|
+
}
|
|
108
|
+
const client = await this.db.connect();
|
|
109
|
+
try {
|
|
110
|
+
await client.query("BEGIN");
|
|
111
|
+
const written = await client.query(`INSERT INTO shared_lake_datasets
|
|
112
|
+
(dataset_id, workspace_id, dataset_key, owner_product, source_product,
|
|
113
|
+
metadata_schema, data_path, representation, typed_table, row_product_column,
|
|
114
|
+
schema_version, classification, lifecycle, last_committed_version, created_by, updated_by)
|
|
115
|
+
VALUES ($1::uuid,$2::uuid,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$15)
|
|
116
|
+
ON CONFLICT (dataset_id) DO UPDATE SET
|
|
117
|
+
dataset_key=EXCLUDED.dataset_key, metadata_schema=EXCLUDED.metadata_schema,
|
|
118
|
+
data_path=EXCLUDED.data_path, classification=EXCLUDED.classification,
|
|
119
|
+
lifecycle=EXCLUDED.lifecycle,
|
|
120
|
+
last_committed_version=GREATEST(shared_lake_datasets.last_committed_version, EXCLUDED.last_committed_version),
|
|
121
|
+
updated_by=EXCLUDED.updated_by, updated_at=now()
|
|
122
|
+
WHERE shared_lake_datasets.workspace_id=EXCLUDED.workspace_id
|
|
123
|
+
AND shared_lake_datasets.owner_product=EXCLUDED.owner_product
|
|
124
|
+
AND shared_lake_datasets.representation='legacy_payload'
|
|
125
|
+
RETURNING dataset_id`, [
|
|
126
|
+
input.datasetId,
|
|
127
|
+
input.workspaceId,
|
|
128
|
+
input.datasetKey,
|
|
129
|
+
input.ownerProduct,
|
|
130
|
+
input.sourceProduct,
|
|
131
|
+
input.metadataSchema,
|
|
132
|
+
input.dataPath,
|
|
133
|
+
input.representation,
|
|
134
|
+
null,
|
|
135
|
+
null,
|
|
136
|
+
input.schemaVersion,
|
|
137
|
+
input.classification,
|
|
138
|
+
input.lifecycle ?? "ready",
|
|
139
|
+
input.lastCommittedVersion ?? 0,
|
|
140
|
+
actorId
|
|
141
|
+
]);
|
|
142
|
+
if (written.rowCount !== 1) throw new Error("Dataset identity conflicts with another owner or workspace");
|
|
143
|
+
await client.query(`INSERT INTO shared_lake_dataset_consumers (dataset_id, product_id, can_read, can_write, granted_by)
|
|
144
|
+
VALUES ($1::uuid, 'bigconsole', true, true, $2)
|
|
145
|
+
ON CONFLICT (dataset_id, product_id) DO NOTHING`, [
|
|
146
|
+
input.datasetId,
|
|
147
|
+
actorId
|
|
148
|
+
]);
|
|
149
|
+
await client.query("COMMIT");
|
|
150
|
+
} catch (error) {
|
|
151
|
+
await client.query("ROLLBACK");
|
|
152
|
+
throw error;
|
|
153
|
+
} finally {
|
|
154
|
+
client.release();
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
/** Operator-only registration after the typed table and its physical grants exist. */
|
|
158
|
+
async registerTyped(input, actorId) {
|
|
159
|
+
validateLakeDatasetIdentity(input);
|
|
160
|
+
if (input.representation !== "typed_table" || !input.typedTable) throw new Error("A typed table is required");
|
|
161
|
+
const result = await this.db.query(`INSERT INTO shared_lake_datasets
|
|
162
|
+
(dataset_id, workspace_id, dataset_key, owner_product, source_product, metadata_schema,
|
|
163
|
+
data_path, representation, typed_table, row_product_column, schema_version,
|
|
164
|
+
classification, lifecycle, last_committed_version, created_by, updated_by)
|
|
165
|
+
VALUES ($1::uuid,$2::uuid,$3,$4,$5,$6,$7,'typed_table',$8,'product_id',$9,$10,$11,$12,$13,$13)
|
|
166
|
+
ON CONFLICT (dataset_id) DO UPDATE SET
|
|
167
|
+
representation='typed_table', typed_table=EXCLUDED.typed_table,
|
|
168
|
+
row_product_column='product_id', source_product=EXCLUDED.source_product,
|
|
169
|
+
schema_version=EXCLUDED.schema_version,
|
|
170
|
+
classification=EXCLUDED.classification, lifecycle=EXCLUDED.lifecycle,
|
|
171
|
+
last_committed_version=GREATEST(shared_lake_datasets.last_committed_version, EXCLUDED.last_committed_version),
|
|
172
|
+
updated_by=EXCLUDED.updated_by, updated_at=now()
|
|
173
|
+
WHERE shared_lake_datasets.workspace_id=EXCLUDED.workspace_id
|
|
174
|
+
AND shared_lake_datasets.owner_product=EXCLUDED.owner_product
|
|
175
|
+
AND shared_lake_datasets.dataset_key=EXCLUDED.dataset_key
|
|
176
|
+
AND shared_lake_datasets.metadata_schema=EXCLUDED.metadata_schema
|
|
177
|
+
AND shared_lake_datasets.data_path=EXCLUDED.data_path
|
|
178
|
+
RETURNING dataset_id`, [
|
|
179
|
+
input.datasetId,
|
|
180
|
+
input.workspaceId,
|
|
181
|
+
input.datasetKey,
|
|
182
|
+
input.ownerProduct,
|
|
183
|
+
input.sourceProduct,
|
|
184
|
+
input.metadataSchema,
|
|
185
|
+
input.dataPath,
|
|
186
|
+
input.typedTable,
|
|
187
|
+
input.schemaVersion,
|
|
188
|
+
input.classification,
|
|
189
|
+
input.lifecycle ?? "provisioning",
|
|
190
|
+
input.lastCommittedVersion ?? 0,
|
|
191
|
+
actorId
|
|
192
|
+
]);
|
|
193
|
+
if (result.rowCount !== 1) throw new Error("Typed dataset identity conflicts with existing routing");
|
|
194
|
+
}
|
|
195
|
+
/** Catalog grant is activated only after PostgreSQL and S3 grants are applied. */
|
|
196
|
+
async grantConsumer(datasetId, productId, actorId) {
|
|
197
|
+
if (!UUID_RE.test(datasetId) || !PRODUCT_RE.test(productId)) throw new Error("Invalid consumer grant");
|
|
198
|
+
const result = await this.db.query(`INSERT INTO shared_lake_dataset_consumers (dataset_id, product_id, can_read, can_write, granted_by)
|
|
199
|
+
SELECT d.dataset_id, $2, true, false, $3 FROM shared_lake_datasets d
|
|
200
|
+
WHERE d.dataset_id=$1::uuid AND d.representation='typed_table' AND d.lifecycle='ready'
|
|
201
|
+
ON CONFLICT (dataset_id, product_id) DO UPDATE SET
|
|
202
|
+
can_read=true, can_write=false, granted_by=EXCLUDED.granted_by, updated_at=now()
|
|
203
|
+
RETURNING dataset_id`, [
|
|
204
|
+
datasetId,
|
|
205
|
+
productId,
|
|
206
|
+
actorId
|
|
207
|
+
]);
|
|
208
|
+
if (result.rowCount !== 1) throw new Error("Only ready typed datasets may be shared");
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
// src/lake/authorized-reader.ts
|
|
213
|
+
var AuthorizedLakeReader = class {
|
|
214
|
+
static {
|
|
215
|
+
__name(this, "AuthorizedLakeReader");
|
|
216
|
+
}
|
|
217
|
+
registry;
|
|
218
|
+
executor;
|
|
219
|
+
productId;
|
|
220
|
+
constructor(registry, executor, productId) {
|
|
221
|
+
this.registry = registry;
|
|
222
|
+
this.executor = executor;
|
|
223
|
+
this.productId = productId;
|
|
224
|
+
}
|
|
225
|
+
async readRows(workspaceId, datasetId, limit = 100, offset = 0) {
|
|
226
|
+
const dataset = await this.registry.getApprovedById(workspaceId, this.productId, datasetId);
|
|
227
|
+
if (!dataset) return null;
|
|
228
|
+
if (dataset.representation !== "typed_table" || !dataset.typedTable || dataset.rowProductColumn !== "product_id") {
|
|
229
|
+
throw new Error("Dataset does not have an approved typed table");
|
|
230
|
+
}
|
|
231
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 1e3 || !Number.isInteger(offset) || offset < 0 || offset > 1e4) {
|
|
232
|
+
throw new Error("Invalid lake page");
|
|
233
|
+
}
|
|
234
|
+
const rows = await this.executor.queryReadOnly(dataset, `SELECT * FROM lake.${dataset.typedTable}
|
|
235
|
+
WHERE workspace_id = ? AND (product_id = ? OR product_id IS NULL)
|
|
236
|
+
LIMIT ? OFFSET ?`, [
|
|
237
|
+
workspaceId,
|
|
238
|
+
this.productId,
|
|
239
|
+
limit,
|
|
240
|
+
offset
|
|
241
|
+
]);
|
|
242
|
+
return {
|
|
243
|
+
datasetId: dataset.datasetId,
|
|
244
|
+
schemaVersion: dataset.schemaVersion,
|
|
245
|
+
lastCommittedVersion: dataset.lastCommittedVersion,
|
|
246
|
+
rows
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
};
|
|
250
|
+
|
|
251
|
+
export { AuthorizedLakeReader, LakeDatasetRegistry, validateLakeDatasetIdentity };
|
|
252
|
+
//# sourceMappingURL=index.js.map
|
|
253
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/lake/dataset-registry.ts","../../src/lake/authorized-reader.ts"],"names":["UUID_RE","PRODUCT_RE","KEY_RE","TABLE_RE","validateLakeDatasetIdentity","dataset","test","datasetId","workspaceId","Error","datasetKey","ownerProduct","sourceProduct","expectedSchema","replaceAll","metadataSchema","path","URL","dataPath","protocol","hostname","pathname","representation","typedTable","rowProductColumn","toDataset","row","String","schemaVersion","Number","classification","lifecycle","lastCommittedVersion","LakeDatasetRegistry","db","approvedView","approvedSource","approvalPredicate","approvalCheck","getApprovedById","productId","result","query","rows","getApprovedByKey","listApproved","limit","Math","min","max","map","registerLegacy","input","actorId","client","connect","written","rowCount","error","release","registerTyped","grantConsumer","AuthorizedLakeReader","registry","executor","readRows","offset","isInteger","queryReadOnly"],"mappings":";;;AA4BA,IAAMA,OAAAA,GAAU,iEAAA;AAChB,IAAMC,UAAAA,GAAa,wBAAA;AACnB,IAAMC,MAAAA,GAAS,2BAAA;AACf,IAAMC,QAAAA,GAAW,wBAAA;AAEV,SAASC,4BACdC,OAAAA,EAYC;AAED,EAAA,IAAI,CAACL,OAAAA,CAAQM,IAAAA,CAAKD,OAAAA,CAAQE,SAAS,CAAA,IAAK,CAACP,OAAAA,CAAQM,IAAAA,CAAKD,QAAQG,WAAW,CAAA,EACvE,MAAM,IAAIC,MAAM,0BAAA,CAAA;AAClB,EAAA,IAAI,CAACP,OAAOI,IAAAA,CAAKD,OAAAA,CAAQK,UAAU,CAAA,EAAG,MAAM,IAAID,KAAAA,CAAM,qBAAA,CAAA;AACtD,EAAA,IAAI,CAACR,UAAAA,CAAWK,IAAAA,CAAKD,OAAAA,CAAQM,YAAY,CAAA,IAAK,CAACV,UAAAA,CAAWK,IAAAA,CAAKD,OAAAA,CAAQO,aAAa,CAAA,EAAG;AACrF,IAAA,MAAM,IAAIH,MAAM,yBAAA,CAAA;AAClB,EAAA;AACA,EAAA,MAAMI,iBAAiB,CAAA,GAAA,EAAMR,OAAAA,CAAQE,UAAUO,UAAAA,CAAW,GAAA,EAAK,GAAA,CAAA,CAAA,CAAA;AAC/D,EAAA,IAAIT,QAAQU,cAAAA,KAAmBF,cAAAA,EAAgB,MAAM,IAAIJ,MAAM,8CAAA,CAAA;AAC/D,EAAA,MAAMO,IAAAA,GAAO,IAAIC,GAAAA,CAAIZ,OAAAA,CAAQa,QAAQ,CAAA;AACrC,EAAA,IAAIF,IAAAA,CAAKG,QAAAA,KAAa,KAAA,IAAS,CAACH,IAAAA,CAAKI,QAAAA,IAAYJ,IAAAA,CAAKK,QAAAA,KAAa,CAAA,WAAA,EAAchB,OAAAA,CAAQE,SAAS,CAAA,CAAA,CAAA,EAAK;AACrG,IAAA,MAAM,IAAIE,MAAM,uCAAA,CAAA;AAClB,EAAA;AACA,EAAA,IAAIJ,OAAAA,CAAQiB,mBAAmB,aAAA,EAAe;AAC5C,IAAA,IAAI,CAACjB,OAAAA,CAAQkB,UAAAA,IAAc,CAACpB,QAAAA,CAASG,IAAAA,CAAKD,OAAAA,CAAQkB,UAAU,CAAA,EAAG,MAAM,IAAId,KAAAA,CAAM,qBAAA,CAAA;AAC/E,IAAA,IAAIJ,QAAQmB,gBAAAA,KAAqB,YAAA,EAAc,MAAM,IAAIf,MAAM,yCAAA,CAAA;EACjE,CAAA,MAAA,IAAWJ,OAAAA,CAAQkB,UAAAA,IAAclB,OAAAA,CAAQmB,gBAAAA,EAAkB;AACzD,IAAA,MAAM,IAAIf,MAAM,sDAAA,CAAA;AAClB,EAAA;AACF;AAjCgBL,MAAAA,CAAAA,2BAAAA,EAAAA,6BAAAA,CAAAA;AAmChB,SAASqB,UAAUC,GAAAA,EAA4B;AAC7C,EAAA,MAAMrB,OAAAA,GAAuB;IAC3BE,SAAAA,EAAWoB,MAAAA,CAAOD,GAAAA,CAAI,YAAA,CAAa,CAAA;IACnClB,WAAAA,EAAamB,MAAAA,CAAOD,GAAAA,CAAI,cAAA,CAAe,CAAA;IACvChB,UAAAA,EAAYiB,MAAAA,CAAOD,GAAAA,CAAI,aAAA,CAAc,CAAA;IACrCf,YAAAA,EAAcgB,MAAAA,CAAOD,GAAAA,CAAI,eAAA,CAAgB,CAAA;IACzCd,aAAAA,EAAee,MAAAA,CAAOD,GAAAA,CAAI,gBAAA,CAAiB,CAAA;IAC3CX,cAAAA,EAAgBY,MAAAA,CAAOD,GAAAA,CAAI,iBAAA,CAAkB,CAAA;IAC7CR,QAAAA,EAAUS,MAAAA,CAAOD,GAAAA,CAAI,WAAA,CAAY,CAAA;AACjCJ,IAAAA,cAAAA,EAAgBI,IAAI,gBAAA,CAAA;IACpBH,UAAAA,EAAYG,GAAAA,CAAI,aAAA,CAAA,KAAmB,IAAA,GAAO,OAAOC,MAAAA,CAAOD,GAAAA,CAAI,aAAA,CAAc,CAAA;IAC1EF,gBAAAA,EAAkBE,GAAAA,CAAI,oBAAA,CAAA,KAA0B,IAAA,GAAO,OAAOC,MAAAA,CAAOD,GAAAA,CAAI,oBAAA,CAAqB,CAAA;IAC9FE,aAAAA,EAAeC,MAAAA,CAAOH,GAAAA,CAAI,gBAAA,CAAiB,CAAA;AAC3CI,IAAAA,cAAAA,EAAgBJ,IAAI,gBAAA,CAAA;AACpBK,IAAAA,SAAAA,EAAWL,IAAI,WAAA,CAAA;IACfM,oBAAAA,EAAsBH,MAAAA,CAAOH,GAAAA,CAAI,wBAAA,CAAyB;AAC5D,GAAA;AACAtB,EAAAA,2BAAAA,CAA4BC,OAAAA,CAAAA;AAC5B,EAAA,OAAOA,OAAAA;AACT;AAnBSoB,MAAAA,CAAAA,SAAAA,EAAAA,WAAAA,CAAAA;AAsBF,IAAMQ,sBAAN,MAAMA;EA9Db;;;;;EA+DE,WAAA,CACmBC,EAAAA,EACAC,eAAyD,IAAA,EAC1E;SAFiBD,EAAAA,GAAAA,EAAAA;SACAC,YAAAA,GAAAA,YAAAA;AAChB,EAAA;EAEKC,cAAAA,GAAyB;AAC/B,IAAA,OAAO,IAAA,CAAKD,YAAAA,GACR,CAAA,EAAG,IAAA,CAAKA,YAAY,CAAA,EAAA,CAAA,GACpB,4FAAA;AACN,EAAA;EAEQE,iBAAAA,GAA4B;AAClC,IAAA,OAAO,IAAA,CAAKF,eAAe,cAAA,GAAiB,cAAA;AAC9C,EAAA;EAEQG,aAAAA,GAAwB;AAC9B,IAAA,OAAO,IAAA,CAAKH,eAAe,EAAA,GAAK,uBAAA;AAClC,EAAA;EAEA,MAAMI,eAAAA,CAAgB/B,WAAAA,EAAqBgC,SAAAA,EAAmBjC,SAAAA,EAAgD;AAC5G,IAAA,IAAI,CAACP,OAAAA,CAAQM,IAAAA,CAAKE,WAAAA,KAAgB,CAACR,OAAAA,CAAQM,IAAAA,CAAKC,SAAAA,KAAc,CAACN,UAAAA,CAAWK,IAAAA,CAAKkC,SAAAA,GAAY,OAAO,IAAA;AAClG,IAAA,MAAMC,MAAAA,GAAS,MAAM,IAAA,CAAKP,EAAAA,CAAGQ,MAC3B,CAAA,gBAAA,EAAmB,IAAA,CAAKN,gBAAc;;AAE5B,cAAA,EAAA,IAAA,CAAKC,mBAAiB,CAAA,MAAA,EAAW,IAAA,CAAKC,aAAAA,EAAa,CAAA,0BAAA,CAAA,EAC7D;AAAC9B,MAAAA,WAAAA;AAAaD,MAAAA,SAAAA;AAAWiC,MAAAA;AAAU,KAAA,CAAA;AAErC,IAAA,OAAOC,MAAAA,CAAOE,KAAK,CAAA,CAAA,GAAKlB,UAAUgB,MAAAA,CAAOE,IAAAA,CAAK,CAAA,CAAE,CAAA,GAA+B,IAAA;AACjF,EAAA;EAEA,MAAMC,gBAAAA,CAAiBpC,WAAAA,EAAqBgC,SAAAA,EAAmB9B,UAAAA,EAAiD;AAC9G,IAAA,IAAI,CAACV,OAAAA,CAAQM,IAAAA,CAAKE,WAAAA,KAAgB,CAACP,UAAAA,CAAWK,IAAAA,CAAKkC,SAAAA,KAAc,CAACtC,MAAAA,CAAOI,IAAAA,CAAKI,UAAAA,GAAa,OAAO,IAAA;AAClG,IAAA,MAAM+B,MAAAA,GAAS,MAAM,IAAA,CAAKP,EAAAA,CAAGQ,MAC3B,CAAA,gBAAA,EAAmB,IAAA,CAAKN,gBAAc;;AAE5B,cAAA,EAAA,IAAA,CAAKC,mBAAiB,CAAA,MAAA,EAAW,IAAA,CAAKC,aAAAA,EAAa,CAAA,0BAAA,CAAA,EAC7D;AAAC9B,MAAAA,WAAAA;AAAaE,MAAAA,UAAAA;AAAY8B,MAAAA;AAAU,KAAA,CAAA;AAEtC,IAAA,OAAOC,MAAAA,CAAOE,KAAK,CAAA,CAAA,GAAKlB,UAAUgB,MAAAA,CAAOE,IAAAA,CAAK,CAAA,CAAE,CAAA,GAA+B,IAAA;AACjF,EAAA;AAEA,EAAA,MAAME,YAAAA,CAAarC,WAAAA,EAAqBgC,SAAAA,EAAmBM,KAAAA,GAAQ,GAAA,EAA6B;AAC9F,IAAA,IAAI,CAAC9C,OAAAA,CAAQM,IAAAA,CAAKE,WAAAA,CAAAA,IAAgB,CAACP,UAAAA,CAAWK,IAAAA,CAAKkC,SAAAA,CAAAA,EAAY,OAAO,EAAA;AACtE,IAAA,MAAMC,MAAAA,GAAS,MAAM,IAAA,CAAKP,EAAAA,CAAGQ,MAC3B,CAAA,gBAAA,EAAmB,IAAA,CAAKN,gBAAc;AACE,4CAAA,EAAA,IAAA,CAAKC,mBAAiB,CAAA;AACxD,UAAA,EAAA,IAAA,CAAKC,eAAa,CAAA;AAExB,2CAAA,CAAA,EAAA;AAAC9B,MAAAA,WAAAA;AAAagC,MAAAA,SAAAA;AAAWO,MAAAA,IAAAA,CAAKC,IAAID,IAAAA,CAAKE,GAAAA,CAAIH,KAAAA,EAAO,CAAA,GAAI,GAAA;AAAM,KAAA,CAAA;AAE9D,IAAA,OAAOL,OAAOE,IAAAA,CAAKO,GAAAA,CAAI,CAACxB,GAAAA,KAAiCD,SAAAA,CAAUC,GAAAA,CAAAA,CAAAA;AACrE,EAAA;;EAGA,MAAMyB,cAAAA,CAAeC,OAAiCC,OAAAA,EAAgC;AACpFjD,IAAAA,2BAAAA,CAA4BgD,KAAAA,CAAAA;AAC5B,IAAA,IAAIA,KAAAA,CAAM9B,cAAAA,KAAmB,gBAAA,IAAoB8B,KAAAA,CAAMzC,iBAAiB,YAAA,EAAc;AACpF,MAAA,MAAM,IAAIF,MAAM,qDAAA,CAAA;AAClB,IAAA;AACA,IAAA,MAAM6C,MAAAA,GAAS,MAAM,IAAA,CAAKpB,EAAAA,CAAGqB,OAAAA,EAAO;AACpC,IAAA,IAAI;AACF,MAAA,MAAMD,MAAAA,CAAOZ,MAAM,OAAA,CAAA;AACnB,MAAA,MAAMc,OAAAA,GAAU,MAAMF,MAAAA,CAAOZ,KAAAA,CAC3B,CAAA;;;;;;;;;;;;;;AAeA,6BAAA,CAAA,EAAA;QACEU,KAAAA,CAAM7C,SAAAA;QACN6C,KAAAA,CAAM5C,WAAAA;QACN4C,KAAAA,CAAM1C,UAAAA;QACN0C,KAAAA,CAAMzC,YAAAA;QACNyC,KAAAA,CAAMxC,aAAAA;QACNwC,KAAAA,CAAMrC,cAAAA;QACNqC,KAAAA,CAAMlC,QAAAA;QACNkC,KAAAA,CAAM9B,cAAAA;AACN,QAAA,IAAA;AACA,QAAA,IAAA;QACA8B,KAAAA,CAAMxB,aAAAA;QACNwB,KAAAA,CAAMtB,cAAAA;AACNsB,QAAAA,KAAAA,CAAMrB,SAAAA,IAAa,OAAA;AACnBqB,QAAAA,KAAAA,CAAMpB,oBAAAA,IAAwB,CAAA;AAC9BqB,QAAAA;AACD,OAAA,CAAA;AAEH,MAAA,IAAIG,QAAQC,QAAAA,KAAa,CAAA,EAAG,MAAM,IAAIhD,MAAM,4DAAA,CAAA;AAC5C,MAAA,MAAM6C,OAAOZ,KAAAA,CACX,CAAA;;AAGA,wDAAA,CAAA,EAAA;QAACU,KAAAA,CAAM7C,SAAAA;AAAW8C,QAAAA;AAAQ,OAAA,CAAA;AAE5B,MAAA,MAAMC,MAAAA,CAAOZ,MAAM,QAAA,CAAA;AACrB,IAAA,CAAA,CAAA,OAASgB,KAAAA,EAAO;AACd,MAAA,MAAMJ,MAAAA,CAAOZ,MAAM,UAAA,CAAA;AACnB,MAAA,MAAMgB,KAAAA;IACR,CAAA,SAAA;AACEJ,MAAAA,MAAAA,CAAOK,OAAAA,EAAO;AAChB,IAAA;AACF,EAAA;;EAGA,MAAMC,aAAAA,CAAcR,OAAiCC,OAAAA,EAAgC;AACnFjD,IAAAA,2BAAAA,CAA4BgD,KAAAA,CAAAA;AAC5B,IAAA,IAAIA,KAAAA,CAAM9B,mBAAmB,aAAA,IAAiB,CAAC8B,MAAM7B,UAAAA,EAAY,MAAM,IAAId,KAAAA,CAAM,2BAAA,CAAA;AACjF,IAAA,MAAMgC,MAAAA,GAAS,MAAM,IAAA,CAAKP,EAAAA,CAAGQ,KAAAA,CAC3B,CAAA;;;;;;;;;;;;;;;;;AAkBA,2BAAA,CAAA,EAAA;MACEU,KAAAA,CAAM7C,SAAAA;MACN6C,KAAAA,CAAM5C,WAAAA;MACN4C,KAAAA,CAAM1C,UAAAA;MACN0C,KAAAA,CAAMzC,YAAAA;MACNyC,KAAAA,CAAMxC,aAAAA;MACNwC,KAAAA,CAAMrC,cAAAA;MACNqC,KAAAA,CAAMlC,QAAAA;MACNkC,KAAAA,CAAM7B,UAAAA;MACN6B,KAAAA,CAAMxB,aAAAA;MACNwB,KAAAA,CAAMtB,cAAAA;AACNsB,MAAAA,KAAAA,CAAMrB,SAAAA,IAAa,cAAA;AACnBqB,MAAAA,KAAAA,CAAMpB,oBAAAA,IAAwB,CAAA;AAC9BqB,MAAAA;AACD,KAAA,CAAA;AAEH,IAAA,IAAIZ,OAAOgB,QAAAA,KAAa,CAAA,EAAG,MAAM,IAAIhD,MAAM,wDAAA,CAAA;AAC7C,EAAA;;EAGA,MAAMoD,aAAAA,CAActD,SAAAA,EAAmBiC,SAAAA,EAAmBa,OAAAA,EAAgC;AACxF,IAAA,IAAI,CAACrD,OAAAA,CAAQM,IAAAA,CAAKC,SAAAA,CAAAA,IAAc,CAACN,UAAAA,CAAWK,IAAAA,CAAKkC,SAAAA,CAAAA,EAAY,MAAM,IAAI/B,MAAM,wBAAA,CAAA;AAC7E,IAAA,MAAMgC,MAAAA,GAAS,MAAM,IAAA,CAAKP,EAAAA,CAAGQ,KAAAA,CAC3B,CAAA;;;;;AAMA,2BAAA,CAAA,EAAA;AAACnC,MAAAA,SAAAA;AAAWiC,MAAAA,SAAAA;AAAWa,MAAAA;AAAQ,KAAA,CAAA;AAEjC,IAAA,IAAIZ,OAAOgB,QAAAA,KAAa,CAAA,EAAG,MAAM,IAAIhD,MAAM,yCAAA,CAAA;AAC7C,EAAA;AACF;;;AC5OO,IAAMqD,uBAAN,MAAMA;EALb;;;;;;EAME,WAAA,CACmBC,QAAAA,EACAC,UACAxB,SAAAA,EACjB;SAHiBuB,QAAAA,GAAAA,QAAAA;SACAC,QAAAA,GAAAA,QAAAA;SACAxB,SAAAA,GAAAA,SAAAA;AAChB,EAAA;AAEH,EAAA,MAAMyB,SAASzD,WAAAA,EAAqBD,SAAAA,EAAmBuC,KAAAA,GAAQ,GAAA,EAAKoB,SAAS,CAAA,EAAiC;AAC5G,IAAA,MAAM7D,OAAAA,GAAU,MAAM,IAAA,CAAK0D,QAAAA,CAASxB,gBAAgB/B,WAAAA,EAAa,IAAA,CAAKgC,WAAWjC,SAAAA,CAAAA;AACjF,IAAA,IAAI,CAACF,SAAS,OAAO,IAAA;AACrB,IAAA,IAAIA,OAAAA,CAAQiB,mBAAmB,aAAA,IAAiB,CAACjB,QAAQkB,UAAAA,IAAclB,OAAAA,CAAQmB,qBAAqB,YAAA,EAAc;AAChH,MAAA,MAAM,IAAIf,MAAM,+CAAA,CAAA;AAClB,IAAA;AACA,IAAA,IACE,CAACoB,MAAAA,CAAOsC,SAAAA,CAAUrB,KAAAA,CAAAA,IAClBA,QAAQ,CAAA,IACRA,KAAAA,GAAQ,GAAA,IACR,CAACjB,OAAOsC,SAAAA,CAAUD,MAAAA,KAClBA,MAAAA,GAAS,CAAA,IACTA,SAAS,GAAA,EACT;AACA,MAAA,MAAM,IAAIzD,MAAM,mBAAA,CAAA;AAClB,IAAA;AACA,IAAA,MAAMkC,IAAAA,GAAO,MAAM,IAAA,CAAKqB,QAAAA,CAASI,cAC/B/D,OAAAA,EACA,CAAA,mBAAA,EAAsBA,QAAQkB,UAAU;;AAGxC,wBAAA,CAAA,EAAA;AAACf,MAAAA,WAAAA;MAAa,IAAA,CAAKgC,SAAAA;AAAWM,MAAAA,KAAAA;AAAOoB,MAAAA;AAAO,KAAA,CAAA;AAE9C,IAAA,OAAO;AACL3D,MAAAA,SAAAA,EAAWF,OAAAA,CAAQE,SAAAA;AACnBqB,MAAAA,aAAAA,EAAevB,OAAAA,CAAQuB,aAAAA;AACvBI,MAAAA,oBAAAA,EAAsB3B,OAAAA,CAAQ2B,oBAAAA;AAC9BW,MAAAA;AACF,KAAA;AACF,EAAA;AACF","file":"index.js","sourcesContent":["import type { Pool } from 'pg';\n\nexport type DatasetLifecycle = 'provisioning' | 'ready' | 'archived';\nexport type DatasetRepresentation = 'legacy_payload' | 'typed_table';\nexport type DatasetClassification = 'L1_PUBLIC' | 'L2_INTERNAL' | 'L3_CONFIDENTIAL' | 'L4_REGULATED';\n\nexport interface LakeDataset {\n datasetId: string;\n workspaceId: string;\n datasetKey: string;\n ownerProduct: string;\n sourceProduct: string;\n metadataSchema: string;\n dataPath: string;\n representation: DatasetRepresentation;\n typedTable: string | null;\n rowProductColumn: string | null;\n schemaVersion: number;\n classification: DatasetClassification;\n lifecycle: DatasetLifecycle;\n lastCommittedVersion: number;\n}\n\nexport interface RegisterLakeDatasetInput extends Omit<LakeDataset, 'lifecycle' | 'lastCommittedVersion'> {\n lifecycle?: DatasetLifecycle;\n lastCommittedVersion?: number;\n}\n\nconst UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;\nconst PRODUCT_RE = /^[a-z][a-z0-9-]{0,63}$/;\nconst KEY_RE = /^[^\\x00-\\x1f\\x7f]{1,100}$/;\nconst TABLE_RE = /^[a-z][a-z0-9_]{0,62}$/;\n\nexport function validateLakeDatasetIdentity(\n dataset: Pick<\n LakeDataset,\n | 'datasetId'\n | 'workspaceId'\n | 'datasetKey'\n | 'ownerProduct'\n | 'sourceProduct'\n | 'metadataSchema'\n | 'dataPath'\n | 'typedTable'\n | 'rowProductColumn'\n | 'representation'\n >\n): void {\n if (!UUID_RE.test(dataset.datasetId) || !UUID_RE.test(dataset.workspaceId))\n throw new Error('Invalid dataset identity');\n if (!KEY_RE.test(dataset.datasetKey)) throw new Error('Invalid dataset key');\n if (!PRODUCT_RE.test(dataset.ownerProduct) || !PRODUCT_RE.test(dataset.sourceProduct)) {\n throw new Error('Invalid dataset product');\n }\n const expectedSchema = `dl_${dataset.datasetId.replaceAll('-', '_')}`;\n if (dataset.metadataSchema !== expectedSchema) throw new Error('Dataset catalog schema does not match its ID');\n const path = new URL(dataset.dataPath);\n if (path.protocol !== 's3:' || !path.hostname || path.pathname !== `/datasinks/${dataset.datasetId}/`) {\n throw new Error('Dataset S3 path does not match its ID');\n }\n if (dataset.representation === 'typed_table') {\n if (!dataset.typedTable || !TABLE_RE.test(dataset.typedTable)) throw new Error('Invalid typed table');\n if (dataset.rowProductColumn !== 'product_id') throw new Error('Typed shared tables must use product_id');\n } else if (dataset.typedTable || dataset.rowProductColumn) {\n throw new Error('Legacy payload datasets cannot declare a typed table');\n }\n}\n\nfunction toDataset(row: Record<string, unknown>): LakeDataset {\n const dataset: LakeDataset = {\n datasetId: String(row['dataset_id']),\n workspaceId: String(row['workspace_id']),\n datasetKey: String(row['dataset_key']),\n ownerProduct: String(row['owner_product']),\n sourceProduct: String(row['source_product']),\n metadataSchema: String(row['metadata_schema']),\n dataPath: String(row['data_path']),\n representation: row['representation'] as DatasetRepresentation,\n typedTable: row['typed_table'] === null ? null : String(row['typed_table']),\n rowProductColumn: row['row_product_column'] === null ? null : String(row['row_product_column']),\n schemaVersion: Number(row['schema_version']),\n classification: row['classification'] as DatasetClassification,\n lifecycle: row['lifecycle'] as DatasetLifecycle,\n lastCommittedVersion: Number(row['last_committed_version']),\n };\n validateLakeDatasetIdentity(dataset);\n return dataset;\n}\n\n/** Registry is the authorization source. DataSinkLakeDataset remains an ingest watermark only. */\nexport class LakeDatasetRegistry {\n constructor(\n private readonly db: Pool,\n private readonly approvedView: 'shared_lake_timecampus_approved' | null = null\n ) {}\n\n private approvedSource(): string {\n return this.approvedView\n ? `${this.approvedView} d`\n : 'shared_lake_datasets d JOIN shared_lake_dataset_consumers c ON c.dataset_id = d.dataset_id';\n }\n\n private approvalPredicate(): string {\n return this.approvedView ? 'd.product_id' : 'c.product_id';\n }\n\n private approvalCheck(): string {\n return this.approvedView ? '' : 'AND c.can_read = true';\n }\n\n async getApprovedById(workspaceId: string, productId: string, datasetId: string): Promise<LakeDataset | null> {\n if (!UUID_RE.test(workspaceId) || !UUID_RE.test(datasetId) || !PRODUCT_RE.test(productId)) return null;\n const result = await this.db.query(\n `SELECT d.* FROM ${this.approvedSource()}\n WHERE d.workspace_id = $1::uuid AND d.dataset_id = $2::uuid\n AND ${this.approvalPredicate()} = $3 ${this.approvalCheck()} AND d.lifecycle = 'ready'`,\n [workspaceId, datasetId, productId]\n );\n return result.rows[0] ? toDataset(result.rows[0] as Record<string, unknown>) : null;\n }\n\n async getApprovedByKey(workspaceId: string, productId: string, datasetKey: string): Promise<LakeDataset | null> {\n if (!UUID_RE.test(workspaceId) || !PRODUCT_RE.test(productId) || !KEY_RE.test(datasetKey)) return null;\n const result = await this.db.query(\n `SELECT d.* FROM ${this.approvedSource()}\n WHERE d.workspace_id = $1::uuid AND d.dataset_key = $2\n AND ${this.approvalPredicate()} = $3 ${this.approvalCheck()} AND d.lifecycle = 'ready'`,\n [workspaceId, datasetKey, productId]\n );\n return result.rows[0] ? toDataset(result.rows[0] as Record<string, unknown>) : null;\n }\n\n async listApproved(workspaceId: string, productId: string, limit = 100): Promise<LakeDataset[]> {\n if (!UUID_RE.test(workspaceId) || !PRODUCT_RE.test(productId)) return [];\n const result = await this.db.query(\n `SELECT d.* FROM ${this.approvedSource()}\n WHERE d.workspace_id = $1::uuid AND ${this.approvalPredicate()} = $2\n ${this.approvalCheck()} AND d.lifecycle = 'ready'\n ORDER BY d.dataset_key ASC LIMIT $3`,\n [workspaceId, productId, Math.min(Math.max(limit, 1), 1000)]\n );\n return result.rows.map((row: Record<string, unknown>) => toDataset(row));\n }\n\n /** BigConsole's legacy sink registration: stable datasetId=sinkId and same Parquet prefix. */\n async registerLegacy(input: RegisterLakeDatasetInput, actorId: string): Promise<void> {\n validateLakeDatasetIdentity(input);\n if (input.representation !== 'legacy_payload' || input.ownerProduct !== 'bigconsole') {\n throw new Error('Only BigConsole legacy sinks may use registerLegacy');\n }\n const client = await this.db.connect();\n try {\n await client.query('BEGIN');\n const written = await client.query(\n `INSERT INTO shared_lake_datasets\n (dataset_id, workspace_id, dataset_key, owner_product, source_product,\n metadata_schema, data_path, representation, typed_table, row_product_column,\n schema_version, classification, lifecycle, last_committed_version, created_by, updated_by)\n VALUES ($1::uuid,$2::uuid,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$15)\n ON CONFLICT (dataset_id) DO UPDATE SET\n dataset_key=EXCLUDED.dataset_key, metadata_schema=EXCLUDED.metadata_schema,\n data_path=EXCLUDED.data_path, classification=EXCLUDED.classification,\n lifecycle=EXCLUDED.lifecycle,\n last_committed_version=GREATEST(shared_lake_datasets.last_committed_version, EXCLUDED.last_committed_version),\n updated_by=EXCLUDED.updated_by, updated_at=now()\n WHERE shared_lake_datasets.workspace_id=EXCLUDED.workspace_id\n AND shared_lake_datasets.owner_product=EXCLUDED.owner_product\n AND shared_lake_datasets.representation='legacy_payload'\n RETURNING dataset_id`,\n [\n input.datasetId,\n input.workspaceId,\n input.datasetKey,\n input.ownerProduct,\n input.sourceProduct,\n input.metadataSchema,\n input.dataPath,\n input.representation,\n null,\n null,\n input.schemaVersion,\n input.classification,\n input.lifecycle ?? 'ready',\n input.lastCommittedVersion ?? 0,\n actorId,\n ]\n );\n if (written.rowCount !== 1) throw new Error('Dataset identity conflicts with another owner or workspace');\n await client.query(\n `INSERT INTO shared_lake_dataset_consumers (dataset_id, product_id, can_read, can_write, granted_by)\n VALUES ($1::uuid, 'bigconsole', true, true, $2)\n ON CONFLICT (dataset_id, product_id) DO NOTHING`,\n [input.datasetId, actorId]\n );\n await client.query('COMMIT');\n } catch (error) {\n await client.query('ROLLBACK');\n throw error;\n } finally {\n client.release();\n }\n }\n\n /** Operator-only registration after the typed table and its physical grants exist. */\n async registerTyped(input: RegisterLakeDatasetInput, actorId: string): Promise<void> {\n validateLakeDatasetIdentity(input);\n if (input.representation !== 'typed_table' || !input.typedTable) throw new Error('A typed table is required');\n const result = await this.db.query(\n `INSERT INTO shared_lake_datasets\n (dataset_id, workspace_id, dataset_key, owner_product, source_product, metadata_schema,\n data_path, representation, typed_table, row_product_column, schema_version,\n classification, lifecycle, last_committed_version, created_by, updated_by)\n VALUES ($1::uuid,$2::uuid,$3,$4,$5,$6,$7,'typed_table',$8,'product_id',$9,$10,$11,$12,$13,$13)\n ON CONFLICT (dataset_id) DO UPDATE SET\n representation='typed_table', typed_table=EXCLUDED.typed_table,\n row_product_column='product_id', source_product=EXCLUDED.source_product,\n schema_version=EXCLUDED.schema_version,\n classification=EXCLUDED.classification, lifecycle=EXCLUDED.lifecycle,\n last_committed_version=GREATEST(shared_lake_datasets.last_committed_version, EXCLUDED.last_committed_version),\n updated_by=EXCLUDED.updated_by, updated_at=now()\n WHERE shared_lake_datasets.workspace_id=EXCLUDED.workspace_id\n AND shared_lake_datasets.owner_product=EXCLUDED.owner_product\n AND shared_lake_datasets.dataset_key=EXCLUDED.dataset_key\n AND shared_lake_datasets.metadata_schema=EXCLUDED.metadata_schema\n AND shared_lake_datasets.data_path=EXCLUDED.data_path\n RETURNING dataset_id`,\n [\n input.datasetId,\n input.workspaceId,\n input.datasetKey,\n input.ownerProduct,\n input.sourceProduct,\n input.metadataSchema,\n input.dataPath,\n input.typedTable,\n input.schemaVersion,\n input.classification,\n input.lifecycle ?? 'provisioning',\n input.lastCommittedVersion ?? 0,\n actorId,\n ]\n );\n if (result.rowCount !== 1) throw new Error('Typed dataset identity conflicts with existing routing');\n }\n\n /** Catalog grant is activated only after PostgreSQL and S3 grants are applied. */\n async grantConsumer(datasetId: string, productId: string, actorId: string): Promise<void> {\n if (!UUID_RE.test(datasetId) || !PRODUCT_RE.test(productId)) throw new Error('Invalid consumer grant');\n const result = await this.db.query(\n `INSERT INTO shared_lake_dataset_consumers (dataset_id, product_id, can_read, can_write, granted_by)\n SELECT d.dataset_id, $2, true, false, $3 FROM shared_lake_datasets d\n WHERE d.dataset_id=$1::uuid AND d.representation='typed_table' AND d.lifecycle='ready'\n ON CONFLICT (dataset_id, product_id) DO UPDATE SET\n can_read=true, can_write=false, granted_by=EXCLUDED.granted_by, updated_at=now()\n RETURNING dataset_id`,\n [datasetId, productId, actorId]\n );\n if (result.rowCount !== 1) throw new Error('Only ready typed datasets may be shared');\n }\n}\n","import type { LakeDataset, LakeDatasetRegistry } from './dataset-registry.js';\n\n/** An adapter attaches and queries exactly one approved schema atomically. */\nexport interface LakeReadExecutor {\n queryReadOnly(\n dataset: LakeDataset,\n sql: string,\n parameters: readonly (string | number)[]\n ): Promise<Record<string, unknown>[]>;\n}\n\nexport interface LakeReadPage {\n datasetId: string;\n schemaVersion: number;\n lastCommittedVersion: number;\n rows: Record<string, unknown>[];\n}\n\n/**\n * A typed dataset is safe to expose only after registry grant + workspace\n * checks. The row predicate is mandatory even when the dataset is shared.\n * `product_id IS NULL` denotes intentionally shared rows.\n */\nexport class AuthorizedLakeReader {\n constructor(\n private readonly registry: LakeDatasetRegistry,\n private readonly executor: LakeReadExecutor,\n private readonly productId: string\n ) {}\n\n async readRows(workspaceId: string, datasetId: string, limit = 100, offset = 0): Promise<LakeReadPage | null> {\n const dataset = await this.registry.getApprovedById(workspaceId, this.productId, datasetId);\n if (!dataset) return null;\n if (dataset.representation !== 'typed_table' || !dataset.typedTable || dataset.rowProductColumn !== 'product_id') {\n throw new Error('Dataset does not have an approved typed table');\n }\n if (\n !Number.isInteger(limit) ||\n limit < 1 ||\n limit > 1000 ||\n !Number.isInteger(offset) ||\n offset < 0 ||\n offset > 10000\n ) {\n throw new Error('Invalid lake page');\n }\n const rows = await this.executor.queryReadOnly(\n dataset,\n `SELECT * FROM lake.${dataset.typedTable}\n WHERE workspace_id = ? AND (product_id = ? OR product_id IS NULL)\n LIMIT ? OFFSET ?`,\n [workspaceId, this.productId, limit, offset]\n );\n return {\n datasetId: dataset.datasetId,\n schemaVersion: dataset.schemaVersion,\n lastCommittedVersion: dataset.lastCommittedVersion,\n rows,\n };\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@burdenoff/be-sdk",
|
|
3
|
-
"version": "2026.
|
|
3
|
+
"version": "2026.1006.1",
|
|
4
4
|
"description": "Backend SDK for Burdenoff products providing unified access to notifications, RBAC, activity tracking, usage monitoring, and more",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -71,13 +71,15 @@
|
|
|
71
71
|
"LICENSE"
|
|
72
72
|
],
|
|
73
73
|
"scripts": {
|
|
74
|
+
"lake:registry:migrate": "bunx prisma migrate deploy --config prisma.lake.config.ts",
|
|
75
|
+
"lake:grant:timecampus": "bun scripts/lake-grant-timecampus.ts",
|
|
74
76
|
"deps:check": "bun scripts/deps-mode.ts check",
|
|
75
77
|
"deps:npm": "bun scripts/deps-mode.ts npm",
|
|
76
78
|
"deps:status": "bun scripts/deps-mode.ts status",
|
|
77
79
|
"deps:workspace": "bun scripts/deps-mode.ts workspace",
|
|
78
80
|
"dev": "tsx watch ./src/index.ts",
|
|
79
|
-
"build": "bash scripts/with-private-swc-cache.sh env NODE_OPTIONS=--max-old-space-size=4096 tsup src/index.ts src/init/index.ts src/logger/index.ts src/cache/index.ts src/database/index.ts src/event-queue/index.ts src/lifecycle/index.ts src/graph/index.ts src/middleware/index.ts src/observability/index.ts src/graphql/index.ts src/graphql/services/index.ts src/prisma/index.ts src/utils/index.ts src/notifications/index.ts src/tracker/index.ts src/export/index.ts src/dataloader/index.ts src/prisma/generator/index.ts src/server/index.ts src/i18n/index.ts src/instrumentation/index.ts src/rbac/index.ts src/temporal/index.ts src/client/internal-client.ts src/layer/index.ts src/gateway/index.ts src/gateway/security/index.ts src/crypto/index.ts src/feature-flags/index.ts src/telemetry/index.ts src/auth/index.ts src/captcha/index.ts src/metrics/index.ts src/platform/index.ts src/resource-caps/index.ts src/sharding/index.ts src/sharing/index.ts src/validation/index.ts src/product-alias/index.ts --format esm --dts --clean --shims && bun run pack:tgz",
|
|
80
|
-
"build:watch": "bash scripts/with-private-swc-cache.sh tsup src/index.ts src/init/index.ts src/logger/index.ts src/cache/index.ts src/database/index.ts src/event-queue/index.ts src/lifecycle/index.ts src/graph/index.ts src/middleware/index.ts src/observability/index.ts src/graphql/index.ts src/graphql/services/index.ts src/prisma/index.ts src/utils/index.ts src/notifications/index.ts src/tracker/index.ts src/export/index.ts src/dataloader/index.ts src/prisma/generator/index.ts src/server/index.ts src/i18n/index.ts src/instrumentation/index.ts src/rbac/index.ts src/temporal/index.ts src/client/internal-client.ts src/layer/index.ts src/gateway/index.ts src/gateway/security/index.ts src/crypto/index.ts src/feature-flags/index.ts src/telemetry/index.ts src/auth/index.ts src/captcha/index.ts src/metrics/index.ts src/platform/index.ts src/resource-caps/index.ts src/sharding/index.ts src/sharing/index.ts src/validation/index.ts src/product-alias/index.ts --format esm --dts --watch --shims",
|
|
81
|
+
"build": "bash scripts/with-private-swc-cache.sh env NODE_OPTIONS=--max-old-space-size=4096 tsup src/index.ts src/init/index.ts src/logger/index.ts src/cache/index.ts src/database/index.ts src/event-queue/index.ts src/lifecycle/index.ts src/graph/index.ts src/middleware/index.ts src/observability/index.ts src/graphql/index.ts src/graphql/services/index.ts src/prisma/index.ts src/utils/index.ts src/notifications/index.ts src/tracker/index.ts src/export/index.ts src/dataloader/index.ts src/prisma/generator/index.ts src/server/index.ts src/i18n/index.ts src/instrumentation/index.ts src/rbac/index.ts src/temporal/index.ts src/client/internal-client.ts src/layer/index.ts src/gateway/index.ts src/gateway/security/index.ts src/crypto/index.ts src/feature-flags/index.ts src/telemetry/index.ts src/auth/index.ts src/captcha/index.ts src/metrics/index.ts src/platform/index.ts src/resource-caps/index.ts src/sharding/index.ts src/sharing/index.ts src/validation/index.ts src/product-alias/index.ts src/lake/index.ts --format esm --dts --clean --shims && bun run pack:tgz",
|
|
82
|
+
"build:watch": "bash scripts/with-private-swc-cache.sh tsup src/index.ts src/init/index.ts src/logger/index.ts src/cache/index.ts src/database/index.ts src/event-queue/index.ts src/lifecycle/index.ts src/graph/index.ts src/middleware/index.ts src/observability/index.ts src/graphql/index.ts src/graphql/services/index.ts src/prisma/index.ts src/utils/index.ts src/notifications/index.ts src/tracker/index.ts src/export/index.ts src/dataloader/index.ts src/prisma/generator/index.ts src/server/index.ts src/i18n/index.ts src/instrumentation/index.ts src/rbac/index.ts src/temporal/index.ts src/client/internal-client.ts src/layer/index.ts src/gateway/index.ts src/gateway/security/index.ts src/crypto/index.ts src/feature-flags/index.ts src/telemetry/index.ts src/auth/index.ts src/captcha/index.ts src/metrics/index.ts src/platform/index.ts src/resource-caps/index.ts src/sharding/index.ts src/sharing/index.ts src/validation/index.ts src/product-alias/index.ts src/lake/index.ts --format esm --dts --watch --shims",
|
|
81
83
|
"typecheck": "bash scripts/build-limits.sh tsc --noEmit",
|
|
82
84
|
"lint": "eslint ./src",
|
|
83
85
|
"lint:sanity": "if [ \"${CI:-}\" = \"true\" ]; then bun run lint; else bun run lint -- --cache --cache-location .eslintcache; fi",
|
|
@@ -234,6 +236,7 @@
|
|
|
234
236
|
"access": "public"
|
|
235
237
|
},
|
|
236
238
|
"overrides": {
|
|
239
|
+
"@graphql-tools/utils": "12.0.3",
|
|
237
240
|
"hono": "4.13.12",
|
|
238
241
|
"valibot": "1.5.0",
|
|
239
242
|
"protobufjs": "7.6.6",
|