@lightdash/common 2.210.0 → 2.211.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/.tsbuildinfo +1 -1
- package/dist/cjs/compiler/exploreCompiler.d.ts.map +1 -1
- package/dist/cjs/compiler/exploreCompiler.js +8 -4
- package/dist/cjs/compiler/exploreCompiler.js.map +1 -1
- package/dist/cjs/compiler/exploreCompiler.mock.d.ts.map +1 -1
- package/dist/cjs/compiler/exploreCompiler.mock.js +1 -0
- package/dist/cjs/compiler/exploreCompiler.mock.js.map +1 -1
- package/dist/cjs/compiler/joinInflation.d.ts +4 -1
- package/dist/cjs/compiler/joinInflation.d.ts.map +1 -1
- package/dist/cjs/compiler/joinInflation.js.map +1 -1
- package/dist/cjs/compiler/translator.d.ts +4 -4
- package/dist/cjs/compiler/translator.d.ts.map +1 -1
- package/dist/cjs/compiler/translator.js +47 -19
- package/dist/cjs/compiler/translator.js.map +1 -1
- package/dist/cjs/compiler/translator.test.js +260 -0
- package/dist/cjs/compiler/translator.test.js.map +1 -1
- package/dist/cjs/types/dbt.d.ts +2 -0
- package/dist/cjs/types/dbt.d.ts.map +1 -1
- package/dist/cjs/types/dbt.js.map +1 -1
- package/dist/cjs/types/directAccess.d.ts +4 -0
- package/dist/cjs/types/directAccess.d.ts.map +1 -1
- package/dist/cjs/types/table.d.ts +15 -4
- package/dist/cjs/types/table.d.ts.map +1 -1
- package/dist/cjs/types/warehouse.d.ts +26 -0
- package/dist/cjs/types/warehouse.d.ts.map +1 -1
- package/dist/cjs/types/warehouse.js +35 -12
- package/dist/cjs/types/warehouse.js.map +1 -1
- package/dist/cjs/utils/item.d.ts +1 -4
- package/dist/cjs/utils/item.d.ts.map +1 -1
- package/dist/cjs/utils/virtualView.d.ts.map +1 -1
- package/dist/cjs/utils/virtualView.js +1 -0
- package/dist/cjs/utils/virtualView.js.map +1 -1
- package/dist/cjs/utils/virtualView.test.js +1 -0
- package/dist/cjs/utils/virtualView.test.js.map +1 -1
- package/dist/cjs/utils/warehouse.d.ts +3 -1
- package/dist/cjs/utils/warehouse.d.ts.map +1 -1
- package/dist/cjs/utils/warehouse.js +19 -1
- package/dist/cjs/utils/warehouse.js.map +1 -1
- package/dist/esm/.tsbuildinfo +1 -1
- package/dist/esm/compiler/exploreCompiler.d.ts.map +1 -1
- package/dist/esm/compiler/exploreCompiler.js +8 -4
- package/dist/esm/compiler/exploreCompiler.js.map +1 -1
- package/dist/esm/compiler/exploreCompiler.mock.d.ts.map +1 -1
- package/dist/esm/compiler/exploreCompiler.mock.js +1 -0
- package/dist/esm/compiler/exploreCompiler.mock.js.map +1 -1
- package/dist/esm/compiler/joinInflation.d.ts +4 -1
- package/dist/esm/compiler/joinInflation.d.ts.map +1 -1
- package/dist/esm/compiler/joinInflation.js.map +1 -1
- package/dist/esm/compiler/translator.d.ts +4 -4
- package/dist/esm/compiler/translator.d.ts.map +1 -1
- package/dist/esm/compiler/translator.js +48 -20
- package/dist/esm/compiler/translator.js.map +1 -1
- package/dist/esm/compiler/translator.test.js +261 -1
- package/dist/esm/compiler/translator.test.js.map +1 -1
- package/dist/esm/types/dbt.d.ts +2 -0
- package/dist/esm/types/dbt.d.ts.map +1 -1
- package/dist/esm/types/dbt.js.map +1 -1
- package/dist/esm/types/directAccess.d.ts +4 -0
- package/dist/esm/types/directAccess.d.ts.map +1 -1
- package/dist/esm/types/table.d.ts +15 -4
- package/dist/esm/types/table.d.ts.map +1 -1
- package/dist/esm/types/warehouse.d.ts +26 -0
- package/dist/esm/types/warehouse.d.ts.map +1 -1
- package/dist/esm/types/warehouse.js +32 -11
- package/dist/esm/types/warehouse.js.map +1 -1
- package/dist/esm/utils/item.d.ts +1 -4
- package/dist/esm/utils/item.d.ts.map +1 -1
- package/dist/esm/utils/virtualView.d.ts.map +1 -1
- package/dist/esm/utils/virtualView.js +1 -0
- package/dist/esm/utils/virtualView.js.map +1 -1
- package/dist/esm/utils/virtualView.test.js +1 -0
- package/dist/esm/utils/virtualView.test.js.map +1 -1
- package/dist/esm/utils/warehouse.d.ts +3 -1
- package/dist/esm/utils/warehouse.d.ts.map +1 -1
- package/dist/esm/utils/warehouse.js +16 -0
- package/dist/esm/utils/warehouse.js.map +1 -1
- package/dist/types/.tsbuildinfo +1 -1
- package/dist/types/compiler/exploreCompiler.d.ts.map +1 -1
- package/dist/types/compiler/exploreCompiler.js +8 -4
- package/dist/types/compiler/exploreCompiler.js.map +1 -1
- package/dist/types/compiler/exploreCompiler.mock.d.ts.map +1 -1
- package/dist/types/compiler/exploreCompiler.mock.js +1 -0
- package/dist/types/compiler/exploreCompiler.mock.js.map +1 -1
- package/dist/types/compiler/joinInflation.d.ts +4 -1
- package/dist/types/compiler/joinInflation.d.ts.map +1 -1
- package/dist/types/compiler/joinInflation.js.map +1 -1
- package/dist/types/compiler/translator.d.ts +4 -4
- package/dist/types/compiler/translator.d.ts.map +1 -1
- package/dist/types/compiler/translator.js +48 -20
- package/dist/types/compiler/translator.js.map +1 -1
- package/dist/types/compiler/translator.test.js +261 -1
- package/dist/types/compiler/translator.test.js.map +1 -1
- package/dist/types/types/dbt.d.ts +2 -0
- package/dist/types/types/dbt.d.ts.map +1 -1
- package/dist/types/types/dbt.js.map +1 -1
- package/dist/types/types/directAccess.d.ts +4 -0
- package/dist/types/types/directAccess.d.ts.map +1 -1
- package/dist/types/types/table.d.ts +15 -4
- package/dist/types/types/table.d.ts.map +1 -1
- package/dist/types/types/warehouse.d.ts +26 -0
- package/dist/types/types/warehouse.d.ts.map +1 -1
- package/dist/types/types/warehouse.js +32 -11
- package/dist/types/types/warehouse.js.map +1 -1
- package/dist/types/utils/item.d.ts +1 -4
- package/dist/types/utils/item.d.ts.map +1 -1
- package/dist/types/utils/virtualView.d.ts.map +1 -1
- package/dist/types/utils/virtualView.js +1 -0
- package/dist/types/utils/virtualView.js.map +1 -1
- package/dist/types/utils/virtualView.test.js +1 -0
- package/dist/types/utils/virtualView.test.js.map +1 -1
- package/dist/types/utils/warehouse.d.ts +3 -1
- package/dist/types/utils/warehouse.d.ts.map +1 -1
- package/dist/types/utils/warehouse.js +16 -0
- package/dist/types/utils/warehouse.js.map +1 -1
- package/package.json +1 -1
|
@@ -3,7 +3,8 @@ import { getExploreSplitCandidates, InlineErrorType, isExploreError, JoinRelatio
|
|
|
3
3
|
import { DimensionType, FieldType, MetricType, NumberSeparator, } from '../types/field';
|
|
4
4
|
import { DEFAULT_SPOTLIGHT_CONFIG } from '../types/lightdashProjectConfig';
|
|
5
5
|
import { TimeFrames } from '../types/timeFrames';
|
|
6
|
-
import { setCatalogNestedColumnShape, WAREHOUSE_TIMESTAMP_DOMAINS_KEY, } from '../types/warehouse';
|
|
6
|
+
import { setCatalogNestedColumnShape, setCatalogNestedColumnsUnavailable, WAREHOUSE_TIMESTAMP_DOMAINS_KEY, } from '../types/warehouse';
|
|
7
|
+
import { getBigqueryUnnestSql, getDatabricksUnnestSql, } from '../utils/warehouse';
|
|
7
8
|
import { ExploreCompiler } from './exploreCompiler';
|
|
8
9
|
import { warehouseClientMock } from './exploreCompiler.mock';
|
|
9
10
|
import { getExploreParameterDefinitions } from './parameters';
|
|
@@ -2046,6 +2047,7 @@ describe('nested and repeated columns', () => {
|
|
|
2046
2047
|
...warehouseClientMock,
|
|
2047
2048
|
getAdapterType: () => SupportedDbtAdapter.BIGQUERY,
|
|
2048
2049
|
getFieldQuoteChar: () => '`',
|
|
2050
|
+
getUnnestSql: getBigqueryUnnestSql,
|
|
2049
2051
|
};
|
|
2050
2052
|
const nestedColumn = (name, extra = {}) => ({ name, meta: {}, ...extra });
|
|
2051
2053
|
const gaSessions = {
|
|
@@ -2152,6 +2154,9 @@ describe('nested and repeated columns', () => {
|
|
|
2152
2154
|
expect(hits.nestedFrom).toEqual({
|
|
2153
2155
|
parentTable: 'ga_sessions',
|
|
2154
2156
|
columnPath: 'hits',
|
|
2157
|
+
elementSql: '`ga_sessions__hits`',
|
|
2158
|
+
offsetSql: '`ga_sessions__hits__offset`',
|
|
2159
|
+
joinCondition: 'TRUE',
|
|
2155
2160
|
});
|
|
2156
2161
|
expect(hits.sqlTable).toEqual('UNNEST(`ga_sessions`.hits) AS `ga_sessions__hits` WITH OFFSET AS `ga_sessions__hits__offset`');
|
|
2157
2162
|
expect(hits.label).toEqual('Ga sessions: Hits');
|
|
@@ -2174,6 +2179,9 @@ describe('nested and repeated columns', () => {
|
|
|
2174
2179
|
expect(product.nestedFrom).toEqual({
|
|
2175
2180
|
parentTable: 'ga_sessions__hits',
|
|
2176
2181
|
columnPath: 'hits.product',
|
|
2182
|
+
elementSql: '`ga_sessions__hits__product`',
|
|
2183
|
+
offsetSql: '`ga_sessions__hits__product__offset`',
|
|
2184
|
+
joinCondition: 'TRUE',
|
|
2177
2185
|
});
|
|
2178
2186
|
expect(product.sqlTable).toEqual('UNNEST(`ga_sessions__hits`.product) AS `ga_sessions__hits__product` WITH OFFSET AS `ga_sessions__hits__product__offset`');
|
|
2179
2187
|
expect(product.label).toEqual('Ga sessions: Hits: Product');
|
|
@@ -2288,6 +2296,9 @@ describe('nested and repeated columns', () => {
|
|
|
2288
2296
|
expect(firstHits.nestedFrom).toEqual({
|
|
2289
2297
|
parentTable: 'first_session',
|
|
2290
2298
|
columnPath: 'hits',
|
|
2299
|
+
elementSql: '`first_session__hits`',
|
|
2300
|
+
offsetSql: '`first_session__hits__offset`',
|
|
2301
|
+
joinCondition: 'TRUE',
|
|
2291
2302
|
});
|
|
2292
2303
|
expect(firstHits.dimensions['page.pagePath'].compiledSql).toEqual('`first_session__hits`.page.pagePath');
|
|
2293
2304
|
expect(explore.joinedTables.find(({ table }) => table === 'first_session__hits__product')?.tablesReferences).toEqual(['first_session__hits']);
|
|
@@ -2393,6 +2404,9 @@ describe('nested and repeated columns', () => {
|
|
|
2393
2404
|
expect(keywords.nestedFrom).toEqual({
|
|
2394
2405
|
parentTable: 'ga_sessions__hits',
|
|
2395
2406
|
columnPath: 'hits.page.keywords',
|
|
2407
|
+
elementSql: '`ga_sessions__hits__page__keywords`',
|
|
2408
|
+
offsetSql: '`ga_sessions__hits__page__keywords__offset`',
|
|
2409
|
+
joinCondition: 'TRUE',
|
|
2396
2410
|
});
|
|
2397
2411
|
expect(keywords.sqlTable).toEqual('UNNEST(`ga_sessions__hits`.page.keywords) AS `ga_sessions__hits__page__keywords` WITH OFFSET AS `ga_sessions__hits__page__keywords__offset`');
|
|
2398
2412
|
expect(keywords.label).toEqual('Ga sessions: Hits: Page: Keywords');
|
|
@@ -2422,4 +2436,250 @@ describe('nested and repeated columns', () => {
|
|
|
2422
2436
|
expect(JSON.stringify(explore)).toContain('already used by another table');
|
|
2423
2437
|
});
|
|
2424
2438
|
});
|
|
2439
|
+
describe('nested and repeated columns on Databricks', () => {
|
|
2440
|
+
const databricksClientMock = {
|
|
2441
|
+
...warehouseClientMock,
|
|
2442
|
+
getAdapterType: () => SupportedDbtAdapter.DATABRICKS,
|
|
2443
|
+
getFieldQuoteChar: () => '`',
|
|
2444
|
+
getUnnestSql: getDatabricksUnnestSql,
|
|
2445
|
+
};
|
|
2446
|
+
const column = (name, extra = {}) => ({ name, meta: {}, ...extra });
|
|
2447
|
+
const transactions = {
|
|
2448
|
+
...model,
|
|
2449
|
+
name: 'transactions',
|
|
2450
|
+
alias: 'transactions',
|
|
2451
|
+
unique_id: 'model.transactions',
|
|
2452
|
+
database: 'lightdash_staging',
|
|
2453
|
+
schema: 'nested',
|
|
2454
|
+
relation_name: '`lightdash_staging`.`nested`.`transactions`',
|
|
2455
|
+
meta: { primary_key: 'order_id' },
|
|
2456
|
+
columns: {
|
|
2457
|
+
order_id: column('order_id'),
|
|
2458
|
+
amount: column('amount', {
|
|
2459
|
+
meta: { metrics: { total_amount: { type: MetricType.SUM } } },
|
|
2460
|
+
}),
|
|
2461
|
+
'customer.location': column('customer.location'),
|
|
2462
|
+
product: column('product', { description: 'Line items' }),
|
|
2463
|
+
'product.sku': column('product.sku'),
|
|
2464
|
+
'product.price': column('product.price', {
|
|
2465
|
+
meta: {
|
|
2466
|
+
additional_dimensions: {
|
|
2467
|
+
price_x2: {
|
|
2468
|
+
type: DimensionType.NUMBER,
|
|
2469
|
+
sql: '${TABLE}.price * 2',
|
|
2470
|
+
},
|
|
2471
|
+
},
|
|
2472
|
+
metrics: {
|
|
2473
|
+
total_price: { type: MetricType.SUM },
|
|
2474
|
+
total_price_x2: {
|
|
2475
|
+
type: MetricType.SUM,
|
|
2476
|
+
sql: '${TABLE}.price * 2',
|
|
2477
|
+
},
|
|
2478
|
+
},
|
|
2479
|
+
},
|
|
2480
|
+
}),
|
|
2481
|
+
'product.attributes.colour': column('product.attributes.colour'),
|
|
2482
|
+
'product.variants': column('product.variants'),
|
|
2483
|
+
'product.variants.size': column('product.variants.size'),
|
|
2484
|
+
'product.variants.stock': column('product.variants.stock', {
|
|
2485
|
+
meta: { metrics: { total_stock: { type: MetricType.SUM } } },
|
|
2486
|
+
}),
|
|
2487
|
+
tags: column('tags', {
|
|
2488
|
+
meta: { metrics: { last_tag: { type: MetricType.MAX } } },
|
|
2489
|
+
}),
|
|
2490
|
+
labels: column('labels'),
|
|
2491
|
+
},
|
|
2492
|
+
};
|
|
2493
|
+
const catalog = {
|
|
2494
|
+
lightdash_staging: {
|
|
2495
|
+
nested: {
|
|
2496
|
+
transactions: {
|
|
2497
|
+
order_id: DimensionType.STRING,
|
|
2498
|
+
amount: DimensionType.NUMBER,
|
|
2499
|
+
customer: DimensionType.STRING,
|
|
2500
|
+
'customer.id': DimensionType.STRING,
|
|
2501
|
+
'customer.location': DimensionType.STRING,
|
|
2502
|
+
product: DimensionType.STRING,
|
|
2503
|
+
'product.sku': DimensionType.STRING,
|
|
2504
|
+
'product.price': DimensionType.NUMBER,
|
|
2505
|
+
'product.attributes': DimensionType.STRING,
|
|
2506
|
+
'product.attributes.colour': DimensionType.STRING,
|
|
2507
|
+
'product.variants': DimensionType.STRING,
|
|
2508
|
+
'product.variants.size': DimensionType.STRING,
|
|
2509
|
+
'product.variants.stock': DimensionType.NUMBER,
|
|
2510
|
+
tags: DimensionType.STRING,
|
|
2511
|
+
labels: DimensionType.STRING,
|
|
2512
|
+
},
|
|
2513
|
+
},
|
|
2514
|
+
},
|
|
2515
|
+
};
|
|
2516
|
+
Object.entries({
|
|
2517
|
+
customer: { repeated: false, record: true },
|
|
2518
|
+
product: { repeated: true, record: true },
|
|
2519
|
+
'product.attributes': { repeated: false, record: true },
|
|
2520
|
+
'product.variants': { repeated: true, record: true },
|
|
2521
|
+
tags: { repeated: true, record: false },
|
|
2522
|
+
}).forEach(([path, shape]) => setCatalogNestedColumnShape(catalog, 'lightdash_staging', 'nested', 'transactions', path, shape));
|
|
2523
|
+
const typedModels = attachTypesToModels([transactions], catalog, true);
|
|
2524
|
+
const compile = async () => {
|
|
2525
|
+
const explores = await convertExplores(typedModels, false, SupportedDbtAdapter.DATABRICKS, databricksClientMock, { spotlight: DEFAULT_SPOTLIGHT_CONFIG }, { unnestRepeatedColumns: true });
|
|
2526
|
+
const explore = explores.find((e) => e.name === 'transactions');
|
|
2527
|
+
if (!explore || isExploreError(explore)) {
|
|
2528
|
+
throw new Error(JSON.stringify(explore));
|
|
2529
|
+
}
|
|
2530
|
+
return explore;
|
|
2531
|
+
};
|
|
2532
|
+
it('explodes a repeated record laterally and addresses leaves through the element column', async () => {
|
|
2533
|
+
const explore = await compile();
|
|
2534
|
+
expect(Object.keys(explore.tables).sort()).toEqual([
|
|
2535
|
+
'transactions',
|
|
2536
|
+
'transactions__product',
|
|
2537
|
+
'transactions__product__variants',
|
|
2538
|
+
'transactions__tags',
|
|
2539
|
+
]);
|
|
2540
|
+
const base = explore.tables.transactions;
|
|
2541
|
+
expect(base.dimensions['customer.location'].compiledSql).toEqual('`transactions`.customer.location');
|
|
2542
|
+
expect(base.dimensions.labels).toMatchObject({
|
|
2543
|
+
type: DimensionType.STRING,
|
|
2544
|
+
compiledSql: '`transactions`.labels',
|
|
2545
|
+
});
|
|
2546
|
+
const product = explore.tables.transactions__product;
|
|
2547
|
+
expect(product.sqlTable).toEqual('LATERAL posexplode_outer(`transactions`.product) AS `transactions__product`(`transactions__product__offset`, col)');
|
|
2548
|
+
expect(product.nestedFrom).toEqual({
|
|
2549
|
+
parentTable: 'transactions',
|
|
2550
|
+
columnPath: 'product',
|
|
2551
|
+
elementSql: '`transactions__product`.col',
|
|
2552
|
+
offsetSql: '`transactions__product__offset`',
|
|
2553
|
+
joinCondition: null,
|
|
2554
|
+
});
|
|
2555
|
+
expect(product.description).toEqual('Line items');
|
|
2556
|
+
expect(product.dimensions.sku.compiledSql).toEqual('`transactions__product`.col.sku');
|
|
2557
|
+
expect(product.dimensions['attributes.colour'].compiledSql).toEqual('`transactions__product`.col.attributes.colour');
|
|
2558
|
+
expect(product.dimensions.price_x2.compiledSql).toEqual('`transactions__product`.col.price * 2');
|
|
2559
|
+
expect(product.dimensions.offset.compiledSql).toEqual('`transactions__product__offset`');
|
|
2560
|
+
expect(product.metrics.total_price.compiledSql).toEqual('SUM(`transactions__product`.col.price)');
|
|
2561
|
+
expect(product.metrics.total_price_x2.compiledSql).toEqual('SUM(`transactions__product`.col.price * 2)');
|
|
2562
|
+
});
|
|
2563
|
+
it('chains a repeated record inside a repeated record through the parent element', async () => {
|
|
2564
|
+
const explore = await compile();
|
|
2565
|
+
const variants = explore.tables.transactions__product__variants;
|
|
2566
|
+
expect(variants.sqlTable).toEqual('LATERAL posexplode_outer(`transactions__product`.col.variants) AS `transactions__product__variants`(`transactions__product__variants__offset`, col)');
|
|
2567
|
+
expect(variants.nestedFrom).toMatchObject({
|
|
2568
|
+
parentTable: 'transactions__product',
|
|
2569
|
+
columnPath: 'product.variants',
|
|
2570
|
+
elementSql: '`transactions__product__variants`.col',
|
|
2571
|
+
});
|
|
2572
|
+
expect(variants.dimensions.size.compiledSql).toEqual('`transactions__product__variants`.col.size');
|
|
2573
|
+
expect(variants.metrics.total_stock.compiledSql).toEqual('SUM(`transactions__product__variants`.col.stock)');
|
|
2574
|
+
});
|
|
2575
|
+
it('exposes an array of scalars as the element column', async () => {
|
|
2576
|
+
const explore = await compile();
|
|
2577
|
+
const tags = explore.tables.transactions__tags;
|
|
2578
|
+
expect(tags.sqlTable).toEqual('LATERAL posexplode_outer(`transactions`.tags) AS `transactions__tags`(`transactions__tags__offset`, col)');
|
|
2579
|
+
expect(tags.dimensions.value.compiledSql).toEqual('`transactions__tags`.col');
|
|
2580
|
+
expect(tags.metrics.last_tag.compiledSql).toEqual('MAX(`transactions__tags`.col)');
|
|
2581
|
+
});
|
|
2582
|
+
it('keeps the joins one-to-many on TRUE for the inflation detector', async () => {
|
|
2583
|
+
const explore = await compile();
|
|
2584
|
+
expect(explore.joinedTables.map(({ table, type, relationship, compiledSqlOn, tablesReferences, }) => ({
|
|
2585
|
+
table,
|
|
2586
|
+
type,
|
|
2587
|
+
relationship,
|
|
2588
|
+
compiledSqlOn,
|
|
2589
|
+
tablesReferences,
|
|
2590
|
+
}))).toEqual([
|
|
2591
|
+
{
|
|
2592
|
+
table: 'transactions__product',
|
|
2593
|
+
type: 'left',
|
|
2594
|
+
relationship: JoinRelationship.ONE_TO_MANY,
|
|
2595
|
+
compiledSqlOn: 'TRUE',
|
|
2596
|
+
tablesReferences: ['transactions'],
|
|
2597
|
+
},
|
|
2598
|
+
{
|
|
2599
|
+
table: 'transactions__tags',
|
|
2600
|
+
type: 'left',
|
|
2601
|
+
relationship: JoinRelationship.ONE_TO_MANY,
|
|
2602
|
+
compiledSqlOn: 'TRUE',
|
|
2603
|
+
tablesReferences: ['transactions'],
|
|
2604
|
+
},
|
|
2605
|
+
{
|
|
2606
|
+
table: 'transactions__product__variants',
|
|
2607
|
+
type: 'left',
|
|
2608
|
+
relationship: JoinRelationship.ONE_TO_MANY,
|
|
2609
|
+
compiledSqlOn: 'TRUE',
|
|
2610
|
+
tablesReferences: ['transactions__product'],
|
|
2611
|
+
},
|
|
2612
|
+
]);
|
|
2613
|
+
});
|
|
2614
|
+
it('warns instead of unnesting when the warehouse would not describe the nested shape', async () => {
|
|
2615
|
+
const flatCatalog = {
|
|
2616
|
+
lightdash_staging: {
|
|
2617
|
+
nested: {
|
|
2618
|
+
transactions: {
|
|
2619
|
+
order_id: DimensionType.STRING,
|
|
2620
|
+
amount: DimensionType.NUMBER,
|
|
2621
|
+
product: DimensionType.STRING,
|
|
2622
|
+
tags: DimensionType.STRING,
|
|
2623
|
+
labels: DimensionType.STRING,
|
|
2624
|
+
},
|
|
2625
|
+
orders: { id: DimensionType.NUMBER },
|
|
2626
|
+
},
|
|
2627
|
+
},
|
|
2628
|
+
};
|
|
2629
|
+
const reason = 'Databricks did not describe transactions as JSON (PARSE_SYNTAX_ERROR); nested columns need a SQL warehouse or Databricks Runtime 16.2 or newer.';
|
|
2630
|
+
setCatalogNestedColumnsUnavailable(flatCatalog, 'lightdash_staging', 'nested', 'transactions', reason);
|
|
2631
|
+
setCatalogNestedColumnsUnavailable(flatCatalog, 'lightdash_staging', 'nested', 'orders', reason);
|
|
2632
|
+
const flatTransactions = {
|
|
2633
|
+
...transactions,
|
|
2634
|
+
columns: {
|
|
2635
|
+
order_id: column('order_id'),
|
|
2636
|
+
amount: column('amount'),
|
|
2637
|
+
product: column('product'),
|
|
2638
|
+
'product.sku': column('product.sku', {
|
|
2639
|
+
meta: { dimension: { type: DimensionType.STRING } },
|
|
2640
|
+
}),
|
|
2641
|
+
tags: column('tags'),
|
|
2642
|
+
labels: column('labels'),
|
|
2643
|
+
},
|
|
2644
|
+
};
|
|
2645
|
+
const orders = {
|
|
2646
|
+
...transactions,
|
|
2647
|
+
name: 'orders',
|
|
2648
|
+
alias: 'orders',
|
|
2649
|
+
unique_id: 'model.orders',
|
|
2650
|
+
relation_name: '`lightdash_staging`.`nested`.`orders`',
|
|
2651
|
+
meta: {},
|
|
2652
|
+
columns: { id: column('id') },
|
|
2653
|
+
};
|
|
2654
|
+
const explores = await convertExplores(attachTypesToModels([flatTransactions, orders], flatCatalog, false), false, SupportedDbtAdapter.DATABRICKS, databricksClientMock, { spotlight: DEFAULT_SPOTLIGHT_CONFIG }, { unnestRepeatedColumns: true });
|
|
2655
|
+
const explore = explores.find((e) => e.name === 'transactions');
|
|
2656
|
+
if (!explore || isExploreError(explore)) {
|
|
2657
|
+
throw new Error(JSON.stringify(explore));
|
|
2658
|
+
}
|
|
2659
|
+
expect(Object.keys(explore.tables)).toEqual(['transactions']);
|
|
2660
|
+
expect(explore.tables.transactions.dimensions['product.sku'].compiledSql).toEqual('`transactions`.product.sku');
|
|
2661
|
+
expect(explore.tables.transactions.warnings).toEqual([
|
|
2662
|
+
{
|
|
2663
|
+
type: InlineErrorType.WAREHOUSE_COLUMN_ERROR,
|
|
2664
|
+
message: `Nested columns in model "transactions" compile as plain columns: ${reason}`,
|
|
2665
|
+
},
|
|
2666
|
+
]);
|
|
2667
|
+
// A flagged table without dotted columns has nothing to warn about.
|
|
2668
|
+
const ordersExplore = explores.find((e) => e.name === 'orders');
|
|
2669
|
+
if (!ordersExplore || isExploreError(ordersExplore)) {
|
|
2670
|
+
throw new Error(JSON.stringify(ordersExplore));
|
|
2671
|
+
}
|
|
2672
|
+
expect(ordersExplore.tables.orders.warnings).toBeUndefined();
|
|
2673
|
+
});
|
|
2674
|
+
it('fails the model on a warehouse that cannot unnest', async () => {
|
|
2675
|
+
const explores = await convertExplores(typedModels, false, SupportedDbtAdapter.POSTGRES, warehouseClientMock, { spotlight: DEFAULT_SPOTLIGHT_CONFIG }, { unnestRepeatedColumns: true });
|
|
2676
|
+
const explore = explores.find((e) => e.name === 'transactions');
|
|
2677
|
+
if (!explore || !isExploreError(explore)) {
|
|
2678
|
+
throw new Error('expected the model to fail');
|
|
2679
|
+
}
|
|
2680
|
+
expect(explore.errors.map(({ message }) => message)).toEqual([
|
|
2681
|
+
`Repeated column "product" in model "transactions" can't be unnested: postgres doesn't support unnesting repeated columns yet.`,
|
|
2682
|
+
]);
|
|
2683
|
+
});
|
|
2684
|
+
});
|
|
2425
2685
|
//# sourceMappingURL=translator.test.js.map
|