@lightdash/common 2.210.0 → 2.211.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/.tsbuildinfo +1 -1
- package/dist/cjs/compiler/exploreCompiler.d.ts.map +1 -1
- package/dist/cjs/compiler/exploreCompiler.js +8 -4
- package/dist/cjs/compiler/exploreCompiler.js.map +1 -1
- package/dist/cjs/compiler/exploreCompiler.mock.d.ts.map +1 -1
- package/dist/cjs/compiler/exploreCompiler.mock.js +1 -0
- package/dist/cjs/compiler/exploreCompiler.mock.js.map +1 -1
- package/dist/cjs/compiler/joinInflation.d.ts +4 -1
- package/dist/cjs/compiler/joinInflation.d.ts.map +1 -1
- package/dist/cjs/compiler/joinInflation.js.map +1 -1
- package/dist/cjs/compiler/translator.d.ts +4 -4
- package/dist/cjs/compiler/translator.d.ts.map +1 -1
- package/dist/cjs/compiler/translator.js +47 -19
- package/dist/cjs/compiler/translator.js.map +1 -1
- package/dist/cjs/compiler/translator.test.js +260 -0
- package/dist/cjs/compiler/translator.test.js.map +1 -1
- package/dist/cjs/types/dbt.d.ts +2 -0
- package/dist/cjs/types/dbt.d.ts.map +1 -1
- package/dist/cjs/types/dbt.js.map +1 -1
- package/dist/cjs/types/directAccess.d.ts +4 -0
- package/dist/cjs/types/directAccess.d.ts.map +1 -1
- package/dist/cjs/types/table.d.ts +15 -4
- package/dist/cjs/types/table.d.ts.map +1 -1
- package/dist/cjs/types/warehouse.d.ts +26 -0
- package/dist/cjs/types/warehouse.d.ts.map +1 -1
- package/dist/cjs/types/warehouse.js +35 -12
- package/dist/cjs/types/warehouse.js.map +1 -1
- package/dist/cjs/utils/item.d.ts +1 -4
- package/dist/cjs/utils/item.d.ts.map +1 -1
- package/dist/cjs/utils/virtualView.d.ts.map +1 -1
- package/dist/cjs/utils/virtualView.js +1 -0
- package/dist/cjs/utils/virtualView.js.map +1 -1
- package/dist/cjs/utils/virtualView.test.js +1 -0
- package/dist/cjs/utils/virtualView.test.js.map +1 -1
- package/dist/cjs/utils/warehouse.d.ts +3 -1
- package/dist/cjs/utils/warehouse.d.ts.map +1 -1
- package/dist/cjs/utils/warehouse.js +19 -1
- package/dist/cjs/utils/warehouse.js.map +1 -1
- package/dist/esm/.tsbuildinfo +1 -1
- package/dist/esm/compiler/exploreCompiler.d.ts.map +1 -1
- package/dist/esm/compiler/exploreCompiler.js +8 -4
- package/dist/esm/compiler/exploreCompiler.js.map +1 -1
- package/dist/esm/compiler/exploreCompiler.mock.d.ts.map +1 -1
- package/dist/esm/compiler/exploreCompiler.mock.js +1 -0
- package/dist/esm/compiler/exploreCompiler.mock.js.map +1 -1
- package/dist/esm/compiler/joinInflation.d.ts +4 -1
- package/dist/esm/compiler/joinInflation.d.ts.map +1 -1
- package/dist/esm/compiler/joinInflation.js.map +1 -1
- package/dist/esm/compiler/translator.d.ts +4 -4
- package/dist/esm/compiler/translator.d.ts.map +1 -1
- package/dist/esm/compiler/translator.js +48 -20
- package/dist/esm/compiler/translator.js.map +1 -1
- package/dist/esm/compiler/translator.test.js +261 -1
- package/dist/esm/compiler/translator.test.js.map +1 -1
- package/dist/esm/types/dbt.d.ts +2 -0
- package/dist/esm/types/dbt.d.ts.map +1 -1
- package/dist/esm/types/dbt.js.map +1 -1
- package/dist/esm/types/directAccess.d.ts +4 -0
- package/dist/esm/types/directAccess.d.ts.map +1 -1
- package/dist/esm/types/table.d.ts +15 -4
- package/dist/esm/types/table.d.ts.map +1 -1
- package/dist/esm/types/warehouse.d.ts +26 -0
- package/dist/esm/types/warehouse.d.ts.map +1 -1
- package/dist/esm/types/warehouse.js +32 -11
- package/dist/esm/types/warehouse.js.map +1 -1
- package/dist/esm/utils/item.d.ts +1 -4
- package/dist/esm/utils/item.d.ts.map +1 -1
- package/dist/esm/utils/virtualView.d.ts.map +1 -1
- package/dist/esm/utils/virtualView.js +1 -0
- package/dist/esm/utils/virtualView.js.map +1 -1
- package/dist/esm/utils/virtualView.test.js +1 -0
- package/dist/esm/utils/virtualView.test.js.map +1 -1
- package/dist/esm/utils/warehouse.d.ts +3 -1
- package/dist/esm/utils/warehouse.d.ts.map +1 -1
- package/dist/esm/utils/warehouse.js +16 -0
- package/dist/esm/utils/warehouse.js.map +1 -1
- package/dist/types/.tsbuildinfo +1 -1
- package/dist/types/compiler/exploreCompiler.d.ts.map +1 -1
- package/dist/types/compiler/exploreCompiler.js +8 -4
- package/dist/types/compiler/exploreCompiler.js.map +1 -1
- package/dist/types/compiler/exploreCompiler.mock.d.ts.map +1 -1
- package/dist/types/compiler/exploreCompiler.mock.js +1 -0
- package/dist/types/compiler/exploreCompiler.mock.js.map +1 -1
- package/dist/types/compiler/joinInflation.d.ts +4 -1
- package/dist/types/compiler/joinInflation.d.ts.map +1 -1
- package/dist/types/compiler/joinInflation.js.map +1 -1
- package/dist/types/compiler/translator.d.ts +4 -4
- package/dist/types/compiler/translator.d.ts.map +1 -1
- package/dist/types/compiler/translator.js +48 -20
- package/dist/types/compiler/translator.js.map +1 -1
- package/dist/types/compiler/translator.test.js +261 -1
- package/dist/types/compiler/translator.test.js.map +1 -1
- package/dist/types/types/dbt.d.ts +2 -0
- package/dist/types/types/dbt.d.ts.map +1 -1
- package/dist/types/types/dbt.js.map +1 -1
- package/dist/types/types/directAccess.d.ts +4 -0
- package/dist/types/types/directAccess.d.ts.map +1 -1
- package/dist/types/types/table.d.ts +15 -4
- package/dist/types/types/table.d.ts.map +1 -1
- package/dist/types/types/warehouse.d.ts +26 -0
- package/dist/types/types/warehouse.d.ts.map +1 -1
- package/dist/types/types/warehouse.js +32 -11
- package/dist/types/types/warehouse.js.map +1 -1
- package/dist/types/utils/item.d.ts +1 -4
- package/dist/types/utils/item.d.ts.map +1 -1
- package/dist/types/utils/virtualView.d.ts.map +1 -1
- package/dist/types/utils/virtualView.js +1 -0
- package/dist/types/utils/virtualView.js.map +1 -1
- package/dist/types/utils/virtualView.test.js +1 -0
- package/dist/types/utils/virtualView.test.js.map +1 -1
- package/dist/types/utils/warehouse.d.ts +3 -1
- package/dist/types/utils/warehouse.d.ts.map +1 -1
- package/dist/types/utils/warehouse.js +16 -0
- package/dist/types/utils/warehouse.js.map +1 -1
- package/package.json +1 -1
|
@@ -6,6 +6,7 @@ const field_1 = require("../types/field");
|
|
|
6
6
|
const lightdashProjectConfig_1 = require("../types/lightdashProjectConfig");
|
|
7
7
|
const timeFrames_1 = require("../types/timeFrames");
|
|
8
8
|
const warehouse_1 = require("../types/warehouse");
|
|
9
|
+
const warehouse_2 = require("../utils/warehouse");
|
|
9
10
|
const exploreCompiler_1 = require("./exploreCompiler");
|
|
10
11
|
const exploreCompiler_mock_1 = require("./exploreCompiler.mock");
|
|
11
12
|
const parameters_1 = require("./parameters");
|
|
@@ -2048,6 +2049,7 @@ describe('nested and repeated columns', () => {
|
|
|
2048
2049
|
...exploreCompiler_mock_1.warehouseClientMock,
|
|
2049
2050
|
getAdapterType: () => dbt_1.SupportedDbtAdapter.BIGQUERY,
|
|
2050
2051
|
getFieldQuoteChar: () => '`',
|
|
2052
|
+
getUnnestSql: warehouse_2.getBigqueryUnnestSql,
|
|
2051
2053
|
};
|
|
2052
2054
|
const nestedColumn = (name, extra = {}) => ({ name, meta: {}, ...extra });
|
|
2053
2055
|
const gaSessions = {
|
|
@@ -2154,6 +2156,9 @@ describe('nested and repeated columns', () => {
|
|
|
2154
2156
|
expect(hits.nestedFrom).toEqual({
|
|
2155
2157
|
parentTable: 'ga_sessions',
|
|
2156
2158
|
columnPath: 'hits',
|
|
2159
|
+
elementSql: '`ga_sessions__hits`',
|
|
2160
|
+
offsetSql: '`ga_sessions__hits__offset`',
|
|
2161
|
+
joinCondition: 'TRUE',
|
|
2157
2162
|
});
|
|
2158
2163
|
expect(hits.sqlTable).toEqual('UNNEST(`ga_sessions`.hits) AS `ga_sessions__hits` WITH OFFSET AS `ga_sessions__hits__offset`');
|
|
2159
2164
|
expect(hits.label).toEqual('Ga sessions: Hits');
|
|
@@ -2176,6 +2181,9 @@ describe('nested and repeated columns', () => {
|
|
|
2176
2181
|
expect(product.nestedFrom).toEqual({
|
|
2177
2182
|
parentTable: 'ga_sessions__hits',
|
|
2178
2183
|
columnPath: 'hits.product',
|
|
2184
|
+
elementSql: '`ga_sessions__hits__product`',
|
|
2185
|
+
offsetSql: '`ga_sessions__hits__product__offset`',
|
|
2186
|
+
joinCondition: 'TRUE',
|
|
2179
2187
|
});
|
|
2180
2188
|
expect(product.sqlTable).toEqual('UNNEST(`ga_sessions__hits`.product) AS `ga_sessions__hits__product` WITH OFFSET AS `ga_sessions__hits__product__offset`');
|
|
2181
2189
|
expect(product.label).toEqual('Ga sessions: Hits: Product');
|
|
@@ -2290,6 +2298,9 @@ describe('nested and repeated columns', () => {
|
|
|
2290
2298
|
expect(firstHits.nestedFrom).toEqual({
|
|
2291
2299
|
parentTable: 'first_session',
|
|
2292
2300
|
columnPath: 'hits',
|
|
2301
|
+
elementSql: '`first_session__hits`',
|
|
2302
|
+
offsetSql: '`first_session__hits__offset`',
|
|
2303
|
+
joinCondition: 'TRUE',
|
|
2293
2304
|
});
|
|
2294
2305
|
expect(firstHits.dimensions['page.pagePath'].compiledSql).toEqual('`first_session__hits`.page.pagePath');
|
|
2295
2306
|
expect(explore.joinedTables.find(({ table }) => table === 'first_session__hits__product')?.tablesReferences).toEqual(['first_session__hits']);
|
|
@@ -2395,6 +2406,9 @@ describe('nested and repeated columns', () => {
|
|
|
2395
2406
|
expect(keywords.nestedFrom).toEqual({
|
|
2396
2407
|
parentTable: 'ga_sessions__hits',
|
|
2397
2408
|
columnPath: 'hits.page.keywords',
|
|
2409
|
+
elementSql: '`ga_sessions__hits__page__keywords`',
|
|
2410
|
+
offsetSql: '`ga_sessions__hits__page__keywords__offset`',
|
|
2411
|
+
joinCondition: 'TRUE',
|
|
2398
2412
|
});
|
|
2399
2413
|
expect(keywords.sqlTable).toEqual('UNNEST(`ga_sessions__hits`.page.keywords) AS `ga_sessions__hits__page__keywords` WITH OFFSET AS `ga_sessions__hits__page__keywords__offset`');
|
|
2400
2414
|
expect(keywords.label).toEqual('Ga sessions: Hits: Page: Keywords');
|
|
@@ -2424,4 +2438,250 @@ describe('nested and repeated columns', () => {
|
|
|
2424
2438
|
expect(JSON.stringify(explore)).toContain('already used by another table');
|
|
2425
2439
|
});
|
|
2426
2440
|
});
|
|
2441
|
+
describe('nested and repeated columns on Databricks', () => {
|
|
2442
|
+
const databricksClientMock = {
|
|
2443
|
+
...exploreCompiler_mock_1.warehouseClientMock,
|
|
2444
|
+
getAdapterType: () => dbt_1.SupportedDbtAdapter.DATABRICKS,
|
|
2445
|
+
getFieldQuoteChar: () => '`',
|
|
2446
|
+
getUnnestSql: warehouse_2.getDatabricksUnnestSql,
|
|
2447
|
+
};
|
|
2448
|
+
const column = (name, extra = {}) => ({ name, meta: {}, ...extra });
|
|
2449
|
+
const transactions = {
|
|
2450
|
+
...translator_mock_1.model,
|
|
2451
|
+
name: 'transactions',
|
|
2452
|
+
alias: 'transactions',
|
|
2453
|
+
unique_id: 'model.transactions',
|
|
2454
|
+
database: 'lightdash_staging',
|
|
2455
|
+
schema: 'nested',
|
|
2456
|
+
relation_name: '`lightdash_staging`.`nested`.`transactions`',
|
|
2457
|
+
meta: { primary_key: 'order_id' },
|
|
2458
|
+
columns: {
|
|
2459
|
+
order_id: column('order_id'),
|
|
2460
|
+
amount: column('amount', {
|
|
2461
|
+
meta: { metrics: { total_amount: { type: field_1.MetricType.SUM } } },
|
|
2462
|
+
}),
|
|
2463
|
+
'customer.location': column('customer.location'),
|
|
2464
|
+
product: column('product', { description: 'Line items' }),
|
|
2465
|
+
'product.sku': column('product.sku'),
|
|
2466
|
+
'product.price': column('product.price', {
|
|
2467
|
+
meta: {
|
|
2468
|
+
additional_dimensions: {
|
|
2469
|
+
price_x2: {
|
|
2470
|
+
type: field_1.DimensionType.NUMBER,
|
|
2471
|
+
sql: '${TABLE}.price * 2',
|
|
2472
|
+
},
|
|
2473
|
+
},
|
|
2474
|
+
metrics: {
|
|
2475
|
+
total_price: { type: field_1.MetricType.SUM },
|
|
2476
|
+
total_price_x2: {
|
|
2477
|
+
type: field_1.MetricType.SUM,
|
|
2478
|
+
sql: '${TABLE}.price * 2',
|
|
2479
|
+
},
|
|
2480
|
+
},
|
|
2481
|
+
},
|
|
2482
|
+
}),
|
|
2483
|
+
'product.attributes.colour': column('product.attributes.colour'),
|
|
2484
|
+
'product.variants': column('product.variants'),
|
|
2485
|
+
'product.variants.size': column('product.variants.size'),
|
|
2486
|
+
'product.variants.stock': column('product.variants.stock', {
|
|
2487
|
+
meta: { metrics: { total_stock: { type: field_1.MetricType.SUM } } },
|
|
2488
|
+
}),
|
|
2489
|
+
tags: column('tags', {
|
|
2490
|
+
meta: { metrics: { last_tag: { type: field_1.MetricType.MAX } } },
|
|
2491
|
+
}),
|
|
2492
|
+
labels: column('labels'),
|
|
2493
|
+
},
|
|
2494
|
+
};
|
|
2495
|
+
const catalog = {
|
|
2496
|
+
lightdash_staging: {
|
|
2497
|
+
nested: {
|
|
2498
|
+
transactions: {
|
|
2499
|
+
order_id: field_1.DimensionType.STRING,
|
|
2500
|
+
amount: field_1.DimensionType.NUMBER,
|
|
2501
|
+
customer: field_1.DimensionType.STRING,
|
|
2502
|
+
'customer.id': field_1.DimensionType.STRING,
|
|
2503
|
+
'customer.location': field_1.DimensionType.STRING,
|
|
2504
|
+
product: field_1.DimensionType.STRING,
|
|
2505
|
+
'product.sku': field_1.DimensionType.STRING,
|
|
2506
|
+
'product.price': field_1.DimensionType.NUMBER,
|
|
2507
|
+
'product.attributes': field_1.DimensionType.STRING,
|
|
2508
|
+
'product.attributes.colour': field_1.DimensionType.STRING,
|
|
2509
|
+
'product.variants': field_1.DimensionType.STRING,
|
|
2510
|
+
'product.variants.size': field_1.DimensionType.STRING,
|
|
2511
|
+
'product.variants.stock': field_1.DimensionType.NUMBER,
|
|
2512
|
+
tags: field_1.DimensionType.STRING,
|
|
2513
|
+
labels: field_1.DimensionType.STRING,
|
|
2514
|
+
},
|
|
2515
|
+
},
|
|
2516
|
+
},
|
|
2517
|
+
};
|
|
2518
|
+
Object.entries({
|
|
2519
|
+
customer: { repeated: false, record: true },
|
|
2520
|
+
product: { repeated: true, record: true },
|
|
2521
|
+
'product.attributes': { repeated: false, record: true },
|
|
2522
|
+
'product.variants': { repeated: true, record: true },
|
|
2523
|
+
tags: { repeated: true, record: false },
|
|
2524
|
+
}).forEach(([path, shape]) => (0, warehouse_1.setCatalogNestedColumnShape)(catalog, 'lightdash_staging', 'nested', 'transactions', path, shape));
|
|
2525
|
+
const typedModels = (0, translator_1.attachTypesToModels)([transactions], catalog, true);
|
|
2526
|
+
const compile = async () => {
|
|
2527
|
+
const explores = await (0, translator_1.convertExplores)(typedModels, false, dbt_1.SupportedDbtAdapter.DATABRICKS, databricksClientMock, { spotlight: lightdashProjectConfig_1.DEFAULT_SPOTLIGHT_CONFIG }, { unnestRepeatedColumns: true });
|
|
2528
|
+
const explore = explores.find((e) => e.name === 'transactions');
|
|
2529
|
+
if (!explore || (0, explore_1.isExploreError)(explore)) {
|
|
2530
|
+
throw new Error(JSON.stringify(explore));
|
|
2531
|
+
}
|
|
2532
|
+
return explore;
|
|
2533
|
+
};
|
|
2534
|
+
it('explodes a repeated record laterally and addresses leaves through the element column', async () => {
|
|
2535
|
+
const explore = await compile();
|
|
2536
|
+
expect(Object.keys(explore.tables).sort()).toEqual([
|
|
2537
|
+
'transactions',
|
|
2538
|
+
'transactions__product',
|
|
2539
|
+
'transactions__product__variants',
|
|
2540
|
+
'transactions__tags',
|
|
2541
|
+
]);
|
|
2542
|
+
const base = explore.tables.transactions;
|
|
2543
|
+
expect(base.dimensions['customer.location'].compiledSql).toEqual('`transactions`.customer.location');
|
|
2544
|
+
expect(base.dimensions.labels).toMatchObject({
|
|
2545
|
+
type: field_1.DimensionType.STRING,
|
|
2546
|
+
compiledSql: '`transactions`.labels',
|
|
2547
|
+
});
|
|
2548
|
+
const product = explore.tables.transactions__product;
|
|
2549
|
+
expect(product.sqlTable).toEqual('LATERAL posexplode_outer(`transactions`.product) AS `transactions__product`(`transactions__product__offset`, col)');
|
|
2550
|
+
expect(product.nestedFrom).toEqual({
|
|
2551
|
+
parentTable: 'transactions',
|
|
2552
|
+
columnPath: 'product',
|
|
2553
|
+
elementSql: '`transactions__product`.col',
|
|
2554
|
+
offsetSql: '`transactions__product__offset`',
|
|
2555
|
+
joinCondition: null,
|
|
2556
|
+
});
|
|
2557
|
+
expect(product.description).toEqual('Line items');
|
|
2558
|
+
expect(product.dimensions.sku.compiledSql).toEqual('`transactions__product`.col.sku');
|
|
2559
|
+
expect(product.dimensions['attributes.colour'].compiledSql).toEqual('`transactions__product`.col.attributes.colour');
|
|
2560
|
+
expect(product.dimensions.price_x2.compiledSql).toEqual('`transactions__product`.col.price * 2');
|
|
2561
|
+
expect(product.dimensions.offset.compiledSql).toEqual('`transactions__product__offset`');
|
|
2562
|
+
expect(product.metrics.total_price.compiledSql).toEqual('SUM(`transactions__product`.col.price)');
|
|
2563
|
+
expect(product.metrics.total_price_x2.compiledSql).toEqual('SUM(`transactions__product`.col.price * 2)');
|
|
2564
|
+
});
|
|
2565
|
+
it('chains a repeated record inside a repeated record through the parent element', async () => {
|
|
2566
|
+
const explore = await compile();
|
|
2567
|
+
const variants = explore.tables.transactions__product__variants;
|
|
2568
|
+
expect(variants.sqlTable).toEqual('LATERAL posexplode_outer(`transactions__product`.col.variants) AS `transactions__product__variants`(`transactions__product__variants__offset`, col)');
|
|
2569
|
+
expect(variants.nestedFrom).toMatchObject({
|
|
2570
|
+
parentTable: 'transactions__product',
|
|
2571
|
+
columnPath: 'product.variants',
|
|
2572
|
+
elementSql: '`transactions__product__variants`.col',
|
|
2573
|
+
});
|
|
2574
|
+
expect(variants.dimensions.size.compiledSql).toEqual('`transactions__product__variants`.col.size');
|
|
2575
|
+
expect(variants.metrics.total_stock.compiledSql).toEqual('SUM(`transactions__product__variants`.col.stock)');
|
|
2576
|
+
});
|
|
2577
|
+
it('exposes an array of scalars as the element column', async () => {
|
|
2578
|
+
const explore = await compile();
|
|
2579
|
+
const tags = explore.tables.transactions__tags;
|
|
2580
|
+
expect(tags.sqlTable).toEqual('LATERAL posexplode_outer(`transactions`.tags) AS `transactions__tags`(`transactions__tags__offset`, col)');
|
|
2581
|
+
expect(tags.dimensions.value.compiledSql).toEqual('`transactions__tags`.col');
|
|
2582
|
+
expect(tags.metrics.last_tag.compiledSql).toEqual('MAX(`transactions__tags`.col)');
|
|
2583
|
+
});
|
|
2584
|
+
it('keeps the joins one-to-many on TRUE for the inflation detector', async () => {
|
|
2585
|
+
const explore = await compile();
|
|
2586
|
+
expect(explore.joinedTables.map(({ table, type, relationship, compiledSqlOn, tablesReferences, }) => ({
|
|
2587
|
+
table,
|
|
2588
|
+
type,
|
|
2589
|
+
relationship,
|
|
2590
|
+
compiledSqlOn,
|
|
2591
|
+
tablesReferences,
|
|
2592
|
+
}))).toEqual([
|
|
2593
|
+
{
|
|
2594
|
+
table: 'transactions__product',
|
|
2595
|
+
type: 'left',
|
|
2596
|
+
relationship: explore_1.JoinRelationship.ONE_TO_MANY,
|
|
2597
|
+
compiledSqlOn: 'TRUE',
|
|
2598
|
+
tablesReferences: ['transactions'],
|
|
2599
|
+
},
|
|
2600
|
+
{
|
|
2601
|
+
table: 'transactions__tags',
|
|
2602
|
+
type: 'left',
|
|
2603
|
+
relationship: explore_1.JoinRelationship.ONE_TO_MANY,
|
|
2604
|
+
compiledSqlOn: 'TRUE',
|
|
2605
|
+
tablesReferences: ['transactions'],
|
|
2606
|
+
},
|
|
2607
|
+
{
|
|
2608
|
+
table: 'transactions__product__variants',
|
|
2609
|
+
type: 'left',
|
|
2610
|
+
relationship: explore_1.JoinRelationship.ONE_TO_MANY,
|
|
2611
|
+
compiledSqlOn: 'TRUE',
|
|
2612
|
+
tablesReferences: ['transactions__product'],
|
|
2613
|
+
},
|
|
2614
|
+
]);
|
|
2615
|
+
});
|
|
2616
|
+
it('warns instead of unnesting when the warehouse would not describe the nested shape', async () => {
|
|
2617
|
+
const flatCatalog = {
|
|
2618
|
+
lightdash_staging: {
|
|
2619
|
+
nested: {
|
|
2620
|
+
transactions: {
|
|
2621
|
+
order_id: field_1.DimensionType.STRING,
|
|
2622
|
+
amount: field_1.DimensionType.NUMBER,
|
|
2623
|
+
product: field_1.DimensionType.STRING,
|
|
2624
|
+
tags: field_1.DimensionType.STRING,
|
|
2625
|
+
labels: field_1.DimensionType.STRING,
|
|
2626
|
+
},
|
|
2627
|
+
orders: { id: field_1.DimensionType.NUMBER },
|
|
2628
|
+
},
|
|
2629
|
+
},
|
|
2630
|
+
};
|
|
2631
|
+
const reason = 'Databricks did not describe transactions as JSON (PARSE_SYNTAX_ERROR); nested columns need a SQL warehouse or Databricks Runtime 16.2 or newer.';
|
|
2632
|
+
(0, warehouse_1.setCatalogNestedColumnsUnavailable)(flatCatalog, 'lightdash_staging', 'nested', 'transactions', reason);
|
|
2633
|
+
(0, warehouse_1.setCatalogNestedColumnsUnavailable)(flatCatalog, 'lightdash_staging', 'nested', 'orders', reason);
|
|
2634
|
+
const flatTransactions = {
|
|
2635
|
+
...transactions,
|
|
2636
|
+
columns: {
|
|
2637
|
+
order_id: column('order_id'),
|
|
2638
|
+
amount: column('amount'),
|
|
2639
|
+
product: column('product'),
|
|
2640
|
+
'product.sku': column('product.sku', {
|
|
2641
|
+
meta: { dimension: { type: field_1.DimensionType.STRING } },
|
|
2642
|
+
}),
|
|
2643
|
+
tags: column('tags'),
|
|
2644
|
+
labels: column('labels'),
|
|
2645
|
+
},
|
|
2646
|
+
};
|
|
2647
|
+
const orders = {
|
|
2648
|
+
...transactions,
|
|
2649
|
+
name: 'orders',
|
|
2650
|
+
alias: 'orders',
|
|
2651
|
+
unique_id: 'model.orders',
|
|
2652
|
+
relation_name: '`lightdash_staging`.`nested`.`orders`',
|
|
2653
|
+
meta: {},
|
|
2654
|
+
columns: { id: column('id') },
|
|
2655
|
+
};
|
|
2656
|
+
const explores = await (0, translator_1.convertExplores)((0, translator_1.attachTypesToModels)([flatTransactions, orders], flatCatalog, false), false, dbt_1.SupportedDbtAdapter.DATABRICKS, databricksClientMock, { spotlight: lightdashProjectConfig_1.DEFAULT_SPOTLIGHT_CONFIG }, { unnestRepeatedColumns: true });
|
|
2657
|
+
const explore = explores.find((e) => e.name === 'transactions');
|
|
2658
|
+
if (!explore || (0, explore_1.isExploreError)(explore)) {
|
|
2659
|
+
throw new Error(JSON.stringify(explore));
|
|
2660
|
+
}
|
|
2661
|
+
expect(Object.keys(explore.tables)).toEqual(['transactions']);
|
|
2662
|
+
expect(explore.tables.transactions.dimensions['product.sku'].compiledSql).toEqual('`transactions`.product.sku');
|
|
2663
|
+
expect(explore.tables.transactions.warnings).toEqual([
|
|
2664
|
+
{
|
|
2665
|
+
type: explore_1.InlineErrorType.WAREHOUSE_COLUMN_ERROR,
|
|
2666
|
+
message: `Nested columns in model "transactions" compile as plain columns: ${reason}`,
|
|
2667
|
+
},
|
|
2668
|
+
]);
|
|
2669
|
+
// A flagged table without dotted columns has nothing to warn about.
|
|
2670
|
+
const ordersExplore = explores.find((e) => e.name === 'orders');
|
|
2671
|
+
if (!ordersExplore || (0, explore_1.isExploreError)(ordersExplore)) {
|
|
2672
|
+
throw new Error(JSON.stringify(ordersExplore));
|
|
2673
|
+
}
|
|
2674
|
+
expect(ordersExplore.tables.orders.warnings).toBeUndefined();
|
|
2675
|
+
});
|
|
2676
|
+
it('fails the model on a warehouse that cannot unnest', async () => {
|
|
2677
|
+
const explores = await (0, translator_1.convertExplores)(typedModels, false, dbt_1.SupportedDbtAdapter.POSTGRES, exploreCompiler_mock_1.warehouseClientMock, { spotlight: lightdashProjectConfig_1.DEFAULT_SPOTLIGHT_CONFIG }, { unnestRepeatedColumns: true });
|
|
2678
|
+
const explore = explores.find((e) => e.name === 'transactions');
|
|
2679
|
+
if (!explore || !(0, explore_1.isExploreError)(explore)) {
|
|
2680
|
+
throw new Error('expected the model to fail');
|
|
2681
|
+
}
|
|
2682
|
+
expect(explore.errors.map(({ message }) => message)).toEqual([
|
|
2683
|
+
`Repeated column "product" in model "transactions" can't be unnested: postgres doesn't support unnesting repeated columns yet.`,
|
|
2684
|
+
]);
|
|
2685
|
+
});
|
|
2686
|
+
});
|
|
2427
2687
|
//# sourceMappingURL=translator.test.js.map
|