@lightdash/common 2.209.2 → 2.211.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/.tsbuildinfo +1 -1
- package/dist/cjs/compiler/exploreCompiler.d.ts.map +1 -1
- package/dist/cjs/compiler/exploreCompiler.js +8 -4
- package/dist/cjs/compiler/exploreCompiler.js.map +1 -1
- package/dist/cjs/compiler/exploreCompiler.mock.d.ts.map +1 -1
- package/dist/cjs/compiler/exploreCompiler.mock.js +1 -0
- package/dist/cjs/compiler/exploreCompiler.mock.js.map +1 -1
- package/dist/cjs/compiler/joinInflation.d.ts +4 -1
- package/dist/cjs/compiler/joinInflation.d.ts.map +1 -1
- package/dist/cjs/compiler/joinInflation.js.map +1 -1
- package/dist/cjs/compiler/translator.d.ts +4 -4
- package/dist/cjs/compiler/translator.d.ts.map +1 -1
- package/dist/cjs/compiler/translator.js +47 -19
- package/dist/cjs/compiler/translator.js.map +1 -1
- package/dist/cjs/compiler/translator.test.js +260 -0
- package/dist/cjs/compiler/translator.test.js.map +1 -1
- package/dist/cjs/types/dbt.d.ts +2 -0
- package/dist/cjs/types/dbt.d.ts.map +1 -1
- package/dist/cjs/types/dbt.js.map +1 -1
- package/dist/cjs/types/directAccess.d.ts +4 -0
- package/dist/cjs/types/directAccess.d.ts.map +1 -1
- package/dist/cjs/types/featureFlags.d.ts +5 -2
- package/dist/cjs/types/featureFlags.d.ts.map +1 -1
- package/dist/cjs/types/featureFlags.js +5 -2
- package/dist/cjs/types/featureFlags.js.map +1 -1
- package/dist/cjs/types/table.d.ts +15 -4
- package/dist/cjs/types/table.d.ts.map +1 -1
- package/dist/cjs/types/warehouse.d.ts +26 -0
- package/dist/cjs/types/warehouse.d.ts.map +1 -1
- package/dist/cjs/types/warehouse.js +35 -12
- package/dist/cjs/types/warehouse.js.map +1 -1
- package/dist/cjs/utils/item.d.ts +1 -4
- package/dist/cjs/utils/item.d.ts.map +1 -1
- package/dist/cjs/utils/virtualView.d.ts.map +1 -1
- package/dist/cjs/utils/virtualView.js +1 -0
- package/dist/cjs/utils/virtualView.js.map +1 -1
- package/dist/cjs/utils/virtualView.test.js +1 -0
- package/dist/cjs/utils/virtualView.test.js.map +1 -1
- package/dist/cjs/utils/warehouse.d.ts +3 -1
- package/dist/cjs/utils/warehouse.d.ts.map +1 -1
- package/dist/cjs/utils/warehouse.js +19 -1
- package/dist/cjs/utils/warehouse.js.map +1 -1
- package/dist/esm/.tsbuildinfo +1 -1
- package/dist/esm/compiler/exploreCompiler.d.ts.map +1 -1
- package/dist/esm/compiler/exploreCompiler.js +8 -4
- package/dist/esm/compiler/exploreCompiler.js.map +1 -1
- package/dist/esm/compiler/exploreCompiler.mock.d.ts.map +1 -1
- package/dist/esm/compiler/exploreCompiler.mock.js +1 -0
- package/dist/esm/compiler/exploreCompiler.mock.js.map +1 -1
- package/dist/esm/compiler/joinInflation.d.ts +4 -1
- package/dist/esm/compiler/joinInflation.d.ts.map +1 -1
- package/dist/esm/compiler/joinInflation.js.map +1 -1
- package/dist/esm/compiler/translator.d.ts +4 -4
- package/dist/esm/compiler/translator.d.ts.map +1 -1
- package/dist/esm/compiler/translator.js +48 -20
- package/dist/esm/compiler/translator.js.map +1 -1
- package/dist/esm/compiler/translator.test.js +261 -1
- package/dist/esm/compiler/translator.test.js.map +1 -1
- package/dist/esm/types/dbt.d.ts +2 -0
- package/dist/esm/types/dbt.d.ts.map +1 -1
- package/dist/esm/types/dbt.js.map +1 -1
- package/dist/esm/types/directAccess.d.ts +4 -0
- package/dist/esm/types/directAccess.d.ts.map +1 -1
- package/dist/esm/types/featureFlags.d.ts +5 -2
- package/dist/esm/types/featureFlags.d.ts.map +1 -1
- package/dist/esm/types/featureFlags.js +5 -2
- package/dist/esm/types/featureFlags.js.map +1 -1
- package/dist/esm/types/table.d.ts +15 -4
- package/dist/esm/types/table.d.ts.map +1 -1
- package/dist/esm/types/warehouse.d.ts +26 -0
- package/dist/esm/types/warehouse.d.ts.map +1 -1
- package/dist/esm/types/warehouse.js +32 -11
- package/dist/esm/types/warehouse.js.map +1 -1
- package/dist/esm/utils/item.d.ts +1 -4
- package/dist/esm/utils/item.d.ts.map +1 -1
- package/dist/esm/utils/virtualView.d.ts.map +1 -1
- package/dist/esm/utils/virtualView.js +1 -0
- package/dist/esm/utils/virtualView.js.map +1 -1
- package/dist/esm/utils/virtualView.test.js +1 -0
- package/dist/esm/utils/virtualView.test.js.map +1 -1
- package/dist/esm/utils/warehouse.d.ts +3 -1
- package/dist/esm/utils/warehouse.d.ts.map +1 -1
- package/dist/esm/utils/warehouse.js +16 -0
- package/dist/esm/utils/warehouse.js.map +1 -1
- package/dist/types/.tsbuildinfo +1 -1
- package/dist/types/compiler/exploreCompiler.d.ts.map +1 -1
- package/dist/types/compiler/exploreCompiler.js +8 -4
- package/dist/types/compiler/exploreCompiler.js.map +1 -1
- package/dist/types/compiler/exploreCompiler.mock.d.ts.map +1 -1
- package/dist/types/compiler/exploreCompiler.mock.js +1 -0
- package/dist/types/compiler/exploreCompiler.mock.js.map +1 -1
- package/dist/types/compiler/joinInflation.d.ts +4 -1
- package/dist/types/compiler/joinInflation.d.ts.map +1 -1
- package/dist/types/compiler/joinInflation.js.map +1 -1
- package/dist/types/compiler/translator.d.ts +4 -4
- package/dist/types/compiler/translator.d.ts.map +1 -1
- package/dist/types/compiler/translator.js +48 -20
- package/dist/types/compiler/translator.js.map +1 -1
- package/dist/types/compiler/translator.test.js +261 -1
- package/dist/types/compiler/translator.test.js.map +1 -1
- package/dist/types/types/dbt.d.ts +2 -0
- package/dist/types/types/dbt.d.ts.map +1 -1
- package/dist/types/types/dbt.js.map +1 -1
- package/dist/types/types/directAccess.d.ts +4 -0
- package/dist/types/types/directAccess.d.ts.map +1 -1
- package/dist/types/types/featureFlags.d.ts +5 -2
- package/dist/types/types/featureFlags.d.ts.map +1 -1
- package/dist/types/types/featureFlags.js +5 -2
- package/dist/types/types/featureFlags.js.map +1 -1
- package/dist/types/types/table.d.ts +15 -4
- package/dist/types/types/table.d.ts.map +1 -1
- package/dist/types/types/warehouse.d.ts +26 -0
- package/dist/types/types/warehouse.d.ts.map +1 -1
- package/dist/types/types/warehouse.js +32 -11
- package/dist/types/types/warehouse.js.map +1 -1
- package/dist/types/utils/item.d.ts +1 -4
- package/dist/types/utils/item.d.ts.map +1 -1
- package/dist/types/utils/virtualView.d.ts.map +1 -1
- package/dist/types/utils/virtualView.js +1 -0
- package/dist/types/utils/virtualView.js.map +1 -1
- package/dist/types/utils/virtualView.test.js +1 -0
- package/dist/types/utils/virtualView.test.js.map +1 -1
- package/dist/types/utils/warehouse.d.ts +3 -1
- package/dist/types/utils/warehouse.d.ts.map +1 -1
- package/dist/types/utils/warehouse.js +16 -0
- package/dist/types/utils/warehouse.js.map +1 -1
- package/package.json +1 -1
|
@@ -3,7 +3,8 @@ import { getExploreSplitCandidates, InlineErrorType, isExploreError, JoinRelatio
|
|
|
3
3
|
import { DimensionType, FieldType, MetricType, NumberSeparator, } from '../types/field';
|
|
4
4
|
import { DEFAULT_SPOTLIGHT_CONFIG } from '../types/lightdashProjectConfig';
|
|
5
5
|
import { TimeFrames } from '../types/timeFrames';
|
|
6
|
-
import { setCatalogNestedColumnShape, WAREHOUSE_TIMESTAMP_DOMAINS_KEY, } from '../types/warehouse';
|
|
6
|
+
import { setCatalogNestedColumnShape, setCatalogNestedColumnsUnavailable, WAREHOUSE_TIMESTAMP_DOMAINS_KEY, } from '../types/warehouse';
|
|
7
|
+
import { getBigqueryUnnestSql, getDatabricksUnnestSql, } from '../utils/warehouse';
|
|
7
8
|
import { ExploreCompiler } from './exploreCompiler';
|
|
8
9
|
import { warehouseClientMock } from './exploreCompiler.mock';
|
|
9
10
|
import { getExploreParameterDefinitions } from './parameters';
|
|
@@ -2046,6 +2047,7 @@ describe('nested and repeated columns', () => {
|
|
|
2046
2047
|
...warehouseClientMock,
|
|
2047
2048
|
getAdapterType: () => SupportedDbtAdapter.BIGQUERY,
|
|
2048
2049
|
getFieldQuoteChar: () => '`',
|
|
2050
|
+
getUnnestSql: getBigqueryUnnestSql,
|
|
2049
2051
|
};
|
|
2050
2052
|
const nestedColumn = (name, extra = {}) => ({ name, meta: {}, ...extra });
|
|
2051
2053
|
const gaSessions = {
|
|
@@ -2152,6 +2154,9 @@ describe('nested and repeated columns', () => {
|
|
|
2152
2154
|
expect(hits.nestedFrom).toEqual({
|
|
2153
2155
|
parentTable: 'ga_sessions',
|
|
2154
2156
|
columnPath: 'hits',
|
|
2157
|
+
elementSql: '`ga_sessions__hits`',
|
|
2158
|
+
offsetSql: '`ga_sessions__hits__offset`',
|
|
2159
|
+
joinCondition: 'TRUE',
|
|
2155
2160
|
});
|
|
2156
2161
|
expect(hits.sqlTable).toEqual('UNNEST(`ga_sessions`.hits) AS `ga_sessions__hits` WITH OFFSET AS `ga_sessions__hits__offset`');
|
|
2157
2162
|
expect(hits.label).toEqual('Ga sessions: Hits');
|
|
@@ -2174,6 +2179,9 @@ describe('nested and repeated columns', () => {
|
|
|
2174
2179
|
expect(product.nestedFrom).toEqual({
|
|
2175
2180
|
parentTable: 'ga_sessions__hits',
|
|
2176
2181
|
columnPath: 'hits.product',
|
|
2182
|
+
elementSql: '`ga_sessions__hits__product`',
|
|
2183
|
+
offsetSql: '`ga_sessions__hits__product__offset`',
|
|
2184
|
+
joinCondition: 'TRUE',
|
|
2177
2185
|
});
|
|
2178
2186
|
expect(product.sqlTable).toEqual('UNNEST(`ga_sessions__hits`.product) AS `ga_sessions__hits__product` WITH OFFSET AS `ga_sessions__hits__product__offset`');
|
|
2179
2187
|
expect(product.label).toEqual('Ga sessions: Hits: Product');
|
|
@@ -2288,6 +2296,9 @@ describe('nested and repeated columns', () => {
|
|
|
2288
2296
|
expect(firstHits.nestedFrom).toEqual({
|
|
2289
2297
|
parentTable: 'first_session',
|
|
2290
2298
|
columnPath: 'hits',
|
|
2299
|
+
elementSql: '`first_session__hits`',
|
|
2300
|
+
offsetSql: '`first_session__hits__offset`',
|
|
2301
|
+
joinCondition: 'TRUE',
|
|
2291
2302
|
});
|
|
2292
2303
|
expect(firstHits.dimensions['page.pagePath'].compiledSql).toEqual('`first_session__hits`.page.pagePath');
|
|
2293
2304
|
expect(explore.joinedTables.find(({ table }) => table === 'first_session__hits__product')?.tablesReferences).toEqual(['first_session__hits']);
|
|
@@ -2393,6 +2404,9 @@ describe('nested and repeated columns', () => {
|
|
|
2393
2404
|
expect(keywords.nestedFrom).toEqual({
|
|
2394
2405
|
parentTable: 'ga_sessions__hits',
|
|
2395
2406
|
columnPath: 'hits.page.keywords',
|
|
2407
|
+
elementSql: '`ga_sessions__hits__page__keywords`',
|
|
2408
|
+
offsetSql: '`ga_sessions__hits__page__keywords__offset`',
|
|
2409
|
+
joinCondition: 'TRUE',
|
|
2396
2410
|
});
|
|
2397
2411
|
expect(keywords.sqlTable).toEqual('UNNEST(`ga_sessions__hits`.page.keywords) AS `ga_sessions__hits__page__keywords` WITH OFFSET AS `ga_sessions__hits__page__keywords__offset`');
|
|
2398
2412
|
expect(keywords.label).toEqual('Ga sessions: Hits: Page: Keywords');
|
|
@@ -2422,4 +2436,250 @@ describe('nested and repeated columns', () => {
|
|
|
2422
2436
|
expect(JSON.stringify(explore)).toContain('already used by another table');
|
|
2423
2437
|
});
|
|
2424
2438
|
});
|
|
2439
|
+
describe('nested and repeated columns on Databricks', () => {
|
|
2440
|
+
const databricksClientMock = {
|
|
2441
|
+
...warehouseClientMock,
|
|
2442
|
+
getAdapterType: () => SupportedDbtAdapter.DATABRICKS,
|
|
2443
|
+
getFieldQuoteChar: () => '`',
|
|
2444
|
+
getUnnestSql: getDatabricksUnnestSql,
|
|
2445
|
+
};
|
|
2446
|
+
const column = (name, extra = {}) => ({ name, meta: {}, ...extra });
|
|
2447
|
+
const transactions = {
|
|
2448
|
+
...model,
|
|
2449
|
+
name: 'transactions',
|
|
2450
|
+
alias: 'transactions',
|
|
2451
|
+
unique_id: 'model.transactions',
|
|
2452
|
+
database: 'lightdash_staging',
|
|
2453
|
+
schema: 'nested',
|
|
2454
|
+
relation_name: '`lightdash_staging`.`nested`.`transactions`',
|
|
2455
|
+
meta: { primary_key: 'order_id' },
|
|
2456
|
+
columns: {
|
|
2457
|
+
order_id: column('order_id'),
|
|
2458
|
+
amount: column('amount', {
|
|
2459
|
+
meta: { metrics: { total_amount: { type: MetricType.SUM } } },
|
|
2460
|
+
}),
|
|
2461
|
+
'customer.location': column('customer.location'),
|
|
2462
|
+
product: column('product', { description: 'Line items' }),
|
|
2463
|
+
'product.sku': column('product.sku'),
|
|
2464
|
+
'product.price': column('product.price', {
|
|
2465
|
+
meta: {
|
|
2466
|
+
additional_dimensions: {
|
|
2467
|
+
price_x2: {
|
|
2468
|
+
type: DimensionType.NUMBER,
|
|
2469
|
+
sql: '${TABLE}.price * 2',
|
|
2470
|
+
},
|
|
2471
|
+
},
|
|
2472
|
+
metrics: {
|
|
2473
|
+
total_price: { type: MetricType.SUM },
|
|
2474
|
+
total_price_x2: {
|
|
2475
|
+
type: MetricType.SUM,
|
|
2476
|
+
sql: '${TABLE}.price * 2',
|
|
2477
|
+
},
|
|
2478
|
+
},
|
|
2479
|
+
},
|
|
2480
|
+
}),
|
|
2481
|
+
'product.attributes.colour': column('product.attributes.colour'),
|
|
2482
|
+
'product.variants': column('product.variants'),
|
|
2483
|
+
'product.variants.size': column('product.variants.size'),
|
|
2484
|
+
'product.variants.stock': column('product.variants.stock', {
|
|
2485
|
+
meta: { metrics: { total_stock: { type: MetricType.SUM } } },
|
|
2486
|
+
}),
|
|
2487
|
+
tags: column('tags', {
|
|
2488
|
+
meta: { metrics: { last_tag: { type: MetricType.MAX } } },
|
|
2489
|
+
}),
|
|
2490
|
+
labels: column('labels'),
|
|
2491
|
+
},
|
|
2492
|
+
};
|
|
2493
|
+
const catalog = {
|
|
2494
|
+
lightdash_staging: {
|
|
2495
|
+
nested: {
|
|
2496
|
+
transactions: {
|
|
2497
|
+
order_id: DimensionType.STRING,
|
|
2498
|
+
amount: DimensionType.NUMBER,
|
|
2499
|
+
customer: DimensionType.STRING,
|
|
2500
|
+
'customer.id': DimensionType.STRING,
|
|
2501
|
+
'customer.location': DimensionType.STRING,
|
|
2502
|
+
product: DimensionType.STRING,
|
|
2503
|
+
'product.sku': DimensionType.STRING,
|
|
2504
|
+
'product.price': DimensionType.NUMBER,
|
|
2505
|
+
'product.attributes': DimensionType.STRING,
|
|
2506
|
+
'product.attributes.colour': DimensionType.STRING,
|
|
2507
|
+
'product.variants': DimensionType.STRING,
|
|
2508
|
+
'product.variants.size': DimensionType.STRING,
|
|
2509
|
+
'product.variants.stock': DimensionType.NUMBER,
|
|
2510
|
+
tags: DimensionType.STRING,
|
|
2511
|
+
labels: DimensionType.STRING,
|
|
2512
|
+
},
|
|
2513
|
+
},
|
|
2514
|
+
},
|
|
2515
|
+
};
|
|
2516
|
+
Object.entries({
|
|
2517
|
+
customer: { repeated: false, record: true },
|
|
2518
|
+
product: { repeated: true, record: true },
|
|
2519
|
+
'product.attributes': { repeated: false, record: true },
|
|
2520
|
+
'product.variants': { repeated: true, record: true },
|
|
2521
|
+
tags: { repeated: true, record: false },
|
|
2522
|
+
}).forEach(([path, shape]) => setCatalogNestedColumnShape(catalog, 'lightdash_staging', 'nested', 'transactions', path, shape));
|
|
2523
|
+
const typedModels = attachTypesToModels([transactions], catalog, true);
|
|
2524
|
+
const compile = async () => {
|
|
2525
|
+
const explores = await convertExplores(typedModels, false, SupportedDbtAdapter.DATABRICKS, databricksClientMock, { spotlight: DEFAULT_SPOTLIGHT_CONFIG }, { unnestRepeatedColumns: true });
|
|
2526
|
+
const explore = explores.find((e) => e.name === 'transactions');
|
|
2527
|
+
if (!explore || isExploreError(explore)) {
|
|
2528
|
+
throw new Error(JSON.stringify(explore));
|
|
2529
|
+
}
|
|
2530
|
+
return explore;
|
|
2531
|
+
};
|
|
2532
|
+
it('explodes a repeated record laterally and addresses leaves through the element column', async () => {
|
|
2533
|
+
const explore = await compile();
|
|
2534
|
+
expect(Object.keys(explore.tables).sort()).toEqual([
|
|
2535
|
+
'transactions',
|
|
2536
|
+
'transactions__product',
|
|
2537
|
+
'transactions__product__variants',
|
|
2538
|
+
'transactions__tags',
|
|
2539
|
+
]);
|
|
2540
|
+
const base = explore.tables.transactions;
|
|
2541
|
+
expect(base.dimensions['customer.location'].compiledSql).toEqual('`transactions`.customer.location');
|
|
2542
|
+
expect(base.dimensions.labels).toMatchObject({
|
|
2543
|
+
type: DimensionType.STRING,
|
|
2544
|
+
compiledSql: '`transactions`.labels',
|
|
2545
|
+
});
|
|
2546
|
+
const product = explore.tables.transactions__product;
|
|
2547
|
+
expect(product.sqlTable).toEqual('LATERAL posexplode_outer(`transactions`.product) AS `transactions__product`(`transactions__product__offset`, col)');
|
|
2548
|
+
expect(product.nestedFrom).toEqual({
|
|
2549
|
+
parentTable: 'transactions',
|
|
2550
|
+
columnPath: 'product',
|
|
2551
|
+
elementSql: '`transactions__product`.col',
|
|
2552
|
+
offsetSql: '`transactions__product__offset`',
|
|
2553
|
+
joinCondition: null,
|
|
2554
|
+
});
|
|
2555
|
+
expect(product.description).toEqual('Line items');
|
|
2556
|
+
expect(product.dimensions.sku.compiledSql).toEqual('`transactions__product`.col.sku');
|
|
2557
|
+
expect(product.dimensions['attributes.colour'].compiledSql).toEqual('`transactions__product`.col.attributes.colour');
|
|
2558
|
+
expect(product.dimensions.price_x2.compiledSql).toEqual('`transactions__product`.col.price * 2');
|
|
2559
|
+
expect(product.dimensions.offset.compiledSql).toEqual('`transactions__product__offset`');
|
|
2560
|
+
expect(product.metrics.total_price.compiledSql).toEqual('SUM(`transactions__product`.col.price)');
|
|
2561
|
+
expect(product.metrics.total_price_x2.compiledSql).toEqual('SUM(`transactions__product`.col.price * 2)');
|
|
2562
|
+
});
|
|
2563
|
+
it('chains a repeated record inside a repeated record through the parent element', async () => {
|
|
2564
|
+
const explore = await compile();
|
|
2565
|
+
const variants = explore.tables.transactions__product__variants;
|
|
2566
|
+
expect(variants.sqlTable).toEqual('LATERAL posexplode_outer(`transactions__product`.col.variants) AS `transactions__product__variants`(`transactions__product__variants__offset`, col)');
|
|
2567
|
+
expect(variants.nestedFrom).toMatchObject({
|
|
2568
|
+
parentTable: 'transactions__product',
|
|
2569
|
+
columnPath: 'product.variants',
|
|
2570
|
+
elementSql: '`transactions__product__variants`.col',
|
|
2571
|
+
});
|
|
2572
|
+
expect(variants.dimensions.size.compiledSql).toEqual('`transactions__product__variants`.col.size');
|
|
2573
|
+
expect(variants.metrics.total_stock.compiledSql).toEqual('SUM(`transactions__product__variants`.col.stock)');
|
|
2574
|
+
});
|
|
2575
|
+
it('exposes an array of scalars as the element column', async () => {
|
|
2576
|
+
const explore = await compile();
|
|
2577
|
+
const tags = explore.tables.transactions__tags;
|
|
2578
|
+
expect(tags.sqlTable).toEqual('LATERAL posexplode_outer(`transactions`.tags) AS `transactions__tags`(`transactions__tags__offset`, col)');
|
|
2579
|
+
expect(tags.dimensions.value.compiledSql).toEqual('`transactions__tags`.col');
|
|
2580
|
+
expect(tags.metrics.last_tag.compiledSql).toEqual('MAX(`transactions__tags`.col)');
|
|
2581
|
+
});
|
|
2582
|
+
it('keeps the joins one-to-many on TRUE for the inflation detector', async () => {
|
|
2583
|
+
const explore = await compile();
|
|
2584
|
+
expect(explore.joinedTables.map(({ table, type, relationship, compiledSqlOn, tablesReferences, }) => ({
|
|
2585
|
+
table,
|
|
2586
|
+
type,
|
|
2587
|
+
relationship,
|
|
2588
|
+
compiledSqlOn,
|
|
2589
|
+
tablesReferences,
|
|
2590
|
+
}))).toEqual([
|
|
2591
|
+
{
|
|
2592
|
+
table: 'transactions__product',
|
|
2593
|
+
type: 'left',
|
|
2594
|
+
relationship: JoinRelationship.ONE_TO_MANY,
|
|
2595
|
+
compiledSqlOn: 'TRUE',
|
|
2596
|
+
tablesReferences: ['transactions'],
|
|
2597
|
+
},
|
|
2598
|
+
{
|
|
2599
|
+
table: 'transactions__tags',
|
|
2600
|
+
type: 'left',
|
|
2601
|
+
relationship: JoinRelationship.ONE_TO_MANY,
|
|
2602
|
+
compiledSqlOn: 'TRUE',
|
|
2603
|
+
tablesReferences: ['transactions'],
|
|
2604
|
+
},
|
|
2605
|
+
{
|
|
2606
|
+
table: 'transactions__product__variants',
|
|
2607
|
+
type: 'left',
|
|
2608
|
+
relationship: JoinRelationship.ONE_TO_MANY,
|
|
2609
|
+
compiledSqlOn: 'TRUE',
|
|
2610
|
+
tablesReferences: ['transactions__product'],
|
|
2611
|
+
},
|
|
2612
|
+
]);
|
|
2613
|
+
});
|
|
2614
|
+
it('warns instead of unnesting when the warehouse would not describe the nested shape', async () => {
|
|
2615
|
+
const flatCatalog = {
|
|
2616
|
+
lightdash_staging: {
|
|
2617
|
+
nested: {
|
|
2618
|
+
transactions: {
|
|
2619
|
+
order_id: DimensionType.STRING,
|
|
2620
|
+
amount: DimensionType.NUMBER,
|
|
2621
|
+
product: DimensionType.STRING,
|
|
2622
|
+
tags: DimensionType.STRING,
|
|
2623
|
+
labels: DimensionType.STRING,
|
|
2624
|
+
},
|
|
2625
|
+
orders: { id: DimensionType.NUMBER },
|
|
2626
|
+
},
|
|
2627
|
+
},
|
|
2628
|
+
};
|
|
2629
|
+
const reason = 'Databricks did not describe transactions as JSON (PARSE_SYNTAX_ERROR); nested columns need a SQL warehouse or Databricks Runtime 16.2 or newer.';
|
|
2630
|
+
setCatalogNestedColumnsUnavailable(flatCatalog, 'lightdash_staging', 'nested', 'transactions', reason);
|
|
2631
|
+
setCatalogNestedColumnsUnavailable(flatCatalog, 'lightdash_staging', 'nested', 'orders', reason);
|
|
2632
|
+
const flatTransactions = {
|
|
2633
|
+
...transactions,
|
|
2634
|
+
columns: {
|
|
2635
|
+
order_id: column('order_id'),
|
|
2636
|
+
amount: column('amount'),
|
|
2637
|
+
product: column('product'),
|
|
2638
|
+
'product.sku': column('product.sku', {
|
|
2639
|
+
meta: { dimension: { type: DimensionType.STRING } },
|
|
2640
|
+
}),
|
|
2641
|
+
tags: column('tags'),
|
|
2642
|
+
labels: column('labels'),
|
|
2643
|
+
},
|
|
2644
|
+
};
|
|
2645
|
+
const orders = {
|
|
2646
|
+
...transactions,
|
|
2647
|
+
name: 'orders',
|
|
2648
|
+
alias: 'orders',
|
|
2649
|
+
unique_id: 'model.orders',
|
|
2650
|
+
relation_name: '`lightdash_staging`.`nested`.`orders`',
|
|
2651
|
+
meta: {},
|
|
2652
|
+
columns: { id: column('id') },
|
|
2653
|
+
};
|
|
2654
|
+
const explores = await convertExplores(attachTypesToModels([flatTransactions, orders], flatCatalog, false), false, SupportedDbtAdapter.DATABRICKS, databricksClientMock, { spotlight: DEFAULT_SPOTLIGHT_CONFIG }, { unnestRepeatedColumns: true });
|
|
2655
|
+
const explore = explores.find((e) => e.name === 'transactions');
|
|
2656
|
+
if (!explore || isExploreError(explore)) {
|
|
2657
|
+
throw new Error(JSON.stringify(explore));
|
|
2658
|
+
}
|
|
2659
|
+
expect(Object.keys(explore.tables)).toEqual(['transactions']);
|
|
2660
|
+
expect(explore.tables.transactions.dimensions['product.sku'].compiledSql).toEqual('`transactions`.product.sku');
|
|
2661
|
+
expect(explore.tables.transactions.warnings).toEqual([
|
|
2662
|
+
{
|
|
2663
|
+
type: InlineErrorType.WAREHOUSE_COLUMN_ERROR,
|
|
2664
|
+
message: `Nested columns in model "transactions" compile as plain columns: ${reason}`,
|
|
2665
|
+
},
|
|
2666
|
+
]);
|
|
2667
|
+
// A flagged table without dotted columns has nothing to warn about.
|
|
2668
|
+
const ordersExplore = explores.find((e) => e.name === 'orders');
|
|
2669
|
+
if (!ordersExplore || isExploreError(ordersExplore)) {
|
|
2670
|
+
throw new Error(JSON.stringify(ordersExplore));
|
|
2671
|
+
}
|
|
2672
|
+
expect(ordersExplore.tables.orders.warnings).toBeUndefined();
|
|
2673
|
+
});
|
|
2674
|
+
it('fails the model on a warehouse that cannot unnest', async () => {
|
|
2675
|
+
const explores = await convertExplores(typedModels, false, SupportedDbtAdapter.POSTGRES, warehouseClientMock, { spotlight: DEFAULT_SPOTLIGHT_CONFIG }, { unnestRepeatedColumns: true });
|
|
2676
|
+
const explore = explores.find((e) => e.name === 'transactions');
|
|
2677
|
+
if (!explore || !isExploreError(explore)) {
|
|
2678
|
+
throw new Error('expected the model to fail');
|
|
2679
|
+
}
|
|
2680
|
+
expect(explore.errors.map(({ message }) => message)).toEqual([
|
|
2681
|
+
`Repeated column "product" in model "transactions" can't be unnested: postgres doesn't support unnesting repeated columns yet.`,
|
|
2682
|
+
]);
|
|
2683
|
+
});
|
|
2684
|
+
});
|
|
2425
2685
|
//# sourceMappingURL=translator.test.js.map
|