@lightdash/cli 2.205.1 → 2.206.0

package/dist/handlers/compile.d.ts

@@ -16,6 +16,7 @@ export type CompileHandlerOptions = DbtCompileOptions & {
     combine?: boolean;
 };
 export type CompileProjectResult = {
+    dbtModelNames?: string[];
     explores: (Explore | ExploreError)[];
     isProjectComplete: boolean;
 };

package/dist/handlers/compile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/handlers/compile.ts"],"names":[],"mappings":"AAAA,OAAO,EAOH,OAAO,EACP,YAAY,EAkBf,MAAM,mBAAmB,CAAC;AAqB3B,OAAO,EAAE,iBAAiB,EAA8B,MAAM,eAAe,CAAC;AAsB9E,MAAM,MAAM,qBAAqB,GAAG,iBAAiB,GAAG;IACpD,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,QAAQ,EAAE,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CAAC;IACrC,iBAAiB,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEF,eAAO,MAAM,uBAAuB,YACvB,OAAO,GAAG,YAAY,KAChC,OAIW,CAAC;AAEf,eAAO,MAAM,0BAA0B,YAC1B,OAAO,GAAG,YAAY,KAChC,OAAO,GAAG,YAiBZ,CAAC;AAgTF,eAAO,MAAM,cAAc,YACd,qBAAqB,KAC/B,OAAO,CAAC,oBAAoB,CA2a9B,CAAC;AAEF,eAAO,MAAM,OAAO,YACP,qBAAqB,KAC/B,OAAO,CAAC,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CACO,CAAC;AAE7C,eAAO,MAAM,cAAc,oBACN,qBAAqB,kBAmCzC,CAAC"}
+{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/handlers/compile.ts"],"names":[],"mappings":"AAAA,OAAO,EAOH,OAAO,EACP,YAAY,EAkBf,MAAM,mBAAmB,CAAC;AAqB3B,OAAO,EAAE,iBAAiB,EAA8B,MAAM,eAAe,CAAC;AAsB9E,MAAM,MAAM,qBAAqB,GAAG,iBAAiB,GAAG;IACpD,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CAAC;IACrC,iBAAiB,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEF,eAAO,MAAM,uBAAuB,YACvB,OAAO,GAAG,YAAY,KAChC,OAIW,CAAC;AAEf,eAAO,MAAM,0BAA0B,YAC1B,OAAO,GAAG,YAAY,KAChC,OAAO,GAAG,YAiBZ,CAAC;AAgTF,eAAO,MAAM,cAAc,YACd,qBAAqB,KAC/B,OAAO,CAAC,oBAAoB,CA8b9B,CAAC;AAEF,eAAO,MAAM,OAAO,YACP,qBAAqB,KAC/B,OAAO,CAAC,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CACO,CAAC;AAE7C,eAAO,MAAM,cAAc,oBACN,qBAAqB,kBAmCzC,CAAC"}

package/dist/handlers/compile.js

@@ -239,6 +239,7 @@ const compileProject = async (options) => {
     let explores = null;
     let dbtMetrics = null;
     let isProjectComplete = true;
+    let dbtModelNames;
     explores = await getExploresFromLightdashYmlProject({
         projectDir: absoluteProjectPath,
         lightdashProjectConfig,
@@ -268,6 +269,17 @@ const compileProject = async (options) => {
         const { compiledModelIds, originallySelectedModelIds } = await (0, compile_1.maybeCompileModelsAndJoins)({ targetDir: context.targetDir }, options);
         let manifest = await (0, manifest_1.loadManifest)({ targetDir: context.targetDir });
         const projectManifestModels = (0, common_1.getModelsFromManifest)(manifest);
+        if (!options.combineManifest &&
+            !options.combineManifestProjectUuid &&
+            !projectManifestModels.some((model) => model.lightdash_source_name !== undefined ||
+                model.lightdash_source_uuid !== undefined)) {
+            dbtModelNames = [
+                ...new Set(projectManifestModels.flatMap((model) => [
+                    model.name,
+                    `${model.package_name}__${model.name}`,
+                ])),
+            ];
+        }
         isProjectComplete =
             (0, common_1.getCompiledModels)(projectManifestModels, compiledModelIds)
                 .length === projectManifestModels.length;
@@ -501,7 +513,7 @@ const compileProject = async (options) => {
             durationMs: Date.now() - startTime,
         },
     });
-    return { explores, isProjectComplete };
+    return { explores, isProjectComplete, dbtModelNames };
 };
 exports.compileProject = compileProject;
 const compile = async (options) => (await (0, exports.compileProject)(options)).explores;

package/dist/handlers/compileProject.test.js

@@ -181,6 +181,12 @@ describe('compileProject completeness', () => {
         });
         const result = await (0, compile_1.compileProject)(compileOptions(tempDir));
         expect(result.isProjectComplete).toBe(true);
+        expect(result.dbtModelNames).toEqual([
+            'orders',
+            'test__orders',
+            'countries',
+            'test__countries',
+        ]);
     });
     test('reports a selected subset as incomplete', async () => {
         const projectManifest = manifest({
@@ -194,6 +200,41 @@ describe('compileProject completeness', () => {
         });
         const result = await (0, compile_1.compileProject)(compileOptions(tempDir));
         expect(result.isProjectComplete).toBe(false);
+        expect(result.dbtModelNames).toEqual([
+            'orders',
+            'test__orders',
+            'customers',
+            'test__customers',
+        ]);
+    });
+    test('omits cleanup inventory for YAML projects', async () => {
+        vi.mocked(loader_1.loadLightdashModels).mockResolvedValue([
+            {
+                type: 'model',
+                name: 'orders',
+                sql_from: 'SELECT 1 AS id',
+                dimensions: [
+                    { name: 'id', type: common_1.DimensionType.NUMBER, sql: 'id' },
+                ],
+                sourcePath: 'models/orders.yml',
+            },
+        ]);
+        const result = await (0, compile_1.compileProject)(compileOptions(tempDir));
+        expect(result.dbtModelNames).toBeUndefined();
+        expect(manifest_1.loadManifest).not.toHaveBeenCalled();
+    });
+    test.each(['lightdash_source_name', 'lightdash_source_uuid'])('omits cleanup inventory for a manifest annotated with %s', async (sourceField) => {
+        vi.mocked(manifest_1.loadManifest).mockResolvedValue(manifest({
+            'model.test.orders': dbtNode('model.test.orders', 'model', true, {
+                [sourceField]: 'source-a',
+            }),
+        }));
+        vi.mocked(compile_2.maybeCompileModelsAndJoins).mockResolvedValue({
+            compiledModelIds: ['model.test.orders'],
+            originallySelectedModelIds: undefined,
+        });
+        const result = await (0, compile_1.compileProject)(compileOptions(tempDir));
+        expect(result.dbtModelNames).toBeUndefined();
     });
     test('derives completeness before combining an external manifest', async () => {
         const projectManifest = manifest({
@@ -219,6 +260,7 @@ describe('compileProject completeness', () => {
             'orders',
             'compiled',
         ]);
+        expect(result.dbtModelNames).toBeUndefined();
         expect(apiClient_1.lightdashRawApi).not.toHaveBeenCalled();
         expect(console.info).toHaveBeenCalledWith(expect.stringContaining('Combined external manifest from'));
     });
@@ -257,6 +299,7 @@ describe('compileProject completeness', () => {
             'customers',
         ]);
         expect(result.isProjectComplete).toBe(true);
+        expect(result.dbtModelNames).toBeUndefined();
         expect(apiClient_1.lightdashRawApi).toHaveBeenCalledWith({
             method: 'GET',
             url: '/api/v1/projects/project-uuid/dbt/manifest',

package/dist/handlers/compileWarehouseColumnValidation.test.js

@@ -27,7 +27,7 @@ vi.mock('../lightdash/loader');
 vi.mock('./dbt/compile');
 vi.mock('./dbt/apiClient', async (importOriginal) => {
     const original = await importOriginal();
-    return { ...original, lightdashApi: vi.fn() };
+    return { ...original, lightdashApi: vi.fn().mockResolvedValue(undefined) };
 });
 vi.mock('./dbt/getDbtVersion');
 vi.mock('./dbt/getWarehouseClient');

package/dist/handlers/deploy.d.ts

@@ -24,6 +24,7 @@ type DeployHandlerOptions = DbtCompileOptions & {
 type DeployArgs = DeployHandlerOptions & {
     projectUuid: string;
     complete?: boolean;
+    dbtModelNames?: string[];
 };
 export declare const deploy: (explores: (Explore | ExploreError)[], options: DeployArgs) => Promise<void>;
 export declare const deployHandler: (originalOptions: DeployHandlerOptions) => Promise<void>;

package/dist/handlers/deploy.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"deploy.d.ts","sourceRoot":"","sources":["../../src/handlers/deploy.ts"],"names":[],"mappings":"AAAA,OAAO,EAIH,OAAO,EACP,YAAY,EASf,MAAM,mBAAmB,CAAC;AA2B3B,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AASlD,KAAK,oBAAoB,GAAG,iBAAiB,GAAG;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,wBAAwB,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,KAAK,UAAU,GAAG,oBAAoB,GAAG;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACtB,CAAC;AA6RF,eAAO,MAAM,MAAM,aACL,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,WAC3B,UAAU,KACpB,OAAO,CAAC,IAAI,CA6Id,CAAC;AAiGF,eAAO,MAAM,aAAa,oBAA2B,oBAAoB,kBAuKxE,CAAC"}
+{"version":3,"file":"deploy.d.ts","sourceRoot":"","sources":["../../src/handlers/deploy.ts"],"names":[],"mappings":"AAAA,OAAO,EAIH,OAAO,EACP,YAAY,EASf,MAAM,mBAAmB,CAAC;AA2B3B,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AASlD,KAAK,oBAAoB,GAAG,iBAAiB,GAAG;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,wBAAwB,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,KAAK,UAAU,GAAG,oBAAoB,GAAG;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AA6RF,eAAO,MAAM,MAAM,aACL,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,WAC3B,UAAU,KACpB,OAAO,CAAC,IAAI,CA0Kd,CAAC;AAiGF,eAAO,MAAM,aAAa,oBAA2B,oBAAoB,kBAyKxE,CAAC"}

package/dist/handlers/deploy.js

@@ -184,7 +184,7 @@ const deployBatched = async (explores, options) => {
     const finalizeResponse = await (0, apiClient_1.lightdashApi)({
         method: 'POST',
         url: `/api/v2/projects/${options.projectUuid}/deploy/${sessionUuid}/finalize`,
-        body: JSON.stringify({}),
+        body: JSON.stringify({ dbtModelNames: options.dbtModelNames }),
     });
     globalState_1.default.log(styles.success(`Deploy completed! ${finalizeResponse.exploreCount} explores deployed.`));
     logDeployWarnings(finalizeResponse.warnings);
@@ -243,12 +243,32 @@ const deploy = async (explores, options) => {
         !(await deployBatched(deployableExplores, options));
     if (shouldUseLegacyDeploy) {
         const deployStartTime = Date.now();
-        const deployPayload = JSON.stringify(deployableExplores);
+        const deployPayload = JSON.stringify(options.dbtModelNames === undefined
+            ? deployableExplores
+            : {
+                explores: deployableExplores,
+                dbtModelNames: options.dbtModelNames,
+                complete: options.complete === true,
+            });
         try {
             const deployResponse = await (0, apiClient_1.lightdashApi)({
                 method: 'PUT',
-                url: `/api/v1/projects/${options.projectUuid}/explores?complete=${options.complete === true}`,
+                url: options.dbtModelNames === undefined
+                    ? `/api/v1/projects/${options.projectUuid}/explores?complete=${options.complete === true}`
+                    : `/api/v2/projects/${options.projectUuid}/deploy`,
                 body: deployPayload,
+            }).catch(async (error) => {
+                if (options.dbtModelNames === undefined ||
+                    !(error instanceof common_1.LightdashError) ||
+                    error.statusCode !== 404) {
+                    throw error;
+                }
+                globalState_1.default.log(styles.warning('This server does not support deleted dbt model cleanup; deploying without cleanup. Upgrade the server to enable it.'));
+                return (0, apiClient_1.lightdashApi)({
+                    method: 'PUT',
+                    url: `/api/v1/projects/${options.projectUuid}/explores?complete=${options.complete === true}`,
+                    body: JSON.stringify(deployableExplores),
+                });
             });
             if (deployResponse) {
                 logDeployWarnings(deployResponse.warnings);
@@ -450,7 +470,7 @@ const deployHandler = async (originalOptions) => {
         options.disableTimestampConversion =
             await (0, timestampConversion_1.getProjectDisableTimestampConversion)(options.disableTimestampConversion, existingProjectSelection.projectUuid);
     }
-    const { explores, isProjectComplete } = await (0, compile_1.compileProject)(options);
+    const { explores, isProjectComplete, dbtModelNames } = await (0, compile_1.compileProject)(options);
     let projectUuid;
     if (options.create !== undefined) {
         const project = await createNewProject(executionId, options);
@@ -476,6 +496,7 @@ const deployHandler = async (originalOptions) => {
         ...options,
         projectUuid,
         complete: isProjectComplete,
+        dbtModelNames,
     });
     const serverUrl = config.context?.serverUrl?.replace(/\/$/, '');
     let displayUrl = options.create

package/dist/handlers/deploy.test.js

@@ -1,7 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
 const common_1 = require("@lightdash/common");
 const analytics_1 = require("../analytics/analytics");
+const globalState_1 = tslib_1.__importDefault(require("../globalState"));
 const lightdash_config_1 = require("../lightdash-config");
 const apiClient_1 = require("./dbt/apiClient");
 const deploy_1 = require("./deploy");
@@ -82,6 +84,74 @@ describe('deploy completeness transport', () => {
             body: JSON.stringify([compiledExplore]),
         }));
     });
+    test('sends the full model inventory with a selective single-request deploy', async () => {
+        vi.mocked(apiClient_1.lightdashApi).mockResolvedValue(null);
+        const dbtModelNames = ['orders', 'customers'];
+        await (0, deploy_1.deploy)([compiledExplore], {
+            ...deployOptions,
+            batchedDeploy: false,
+            complete: false,
+            dbtModelNames,
+        });
+        expect(apiClient_1.lightdashApi).toHaveBeenCalledWith({
+            method: 'PUT',
+            url: '/api/v2/projects/project-uuid/deploy',
+            body: JSON.stringify({
+                explores: [compiledExplore],
+                dbtModelNames,
+                complete: false,
+            }),
+        });
+    });
+    test('retries the array payload with a warning when an older server lacks the inventory endpoint', async () => {
+        const warning = vi.spyOn(globalState_1.default, 'log');
+        vi.mocked(apiClient_1.lightdashApi).mockImplementation(async ({ url }) => {
+            if (url === '/api/v2/projects/project-uuid/deploy') {
+                throw new common_1.LightdashError({
+                    message: 'Not found',
+                    name: 'NotFoundError',
+                    statusCode: 404,
+                    data: {},
+                });
+            }
+            return null;
+        });
+        await (0, deploy_1.deploy)([compiledExplore], {
+            ...deployOptions,
+            batchedDeploy: false,
+            dbtModelNames: ['orders', 'customers'],
+        });
+        expect(apiClient_1.lightdashApi).toHaveBeenLastCalledWith({
+            method: 'PUT',
+            url: '/api/v1/projects/project-uuid/explores?complete=false',
+            body: JSON.stringify([compiledExplore]),
+        });
+        expect(warning).toHaveBeenCalledWith(expect.stringContaining('does not support deleted dbt model cleanup'));
+        warning.mockRestore();
+    });
+    test.each([422, 500])('does not retry inventory deployment after HTTP %s', async (statusCode) => {
+        const serverError = new common_1.LightdashError({
+            message: 'Unavailable',
+            name: 'InternalServerError',
+            statusCode,
+            data: {},
+        });
+        vi.mocked(apiClient_1.lightdashApi).mockImplementation(async ({ url }) => {
+            if (url === '/api/v2/projects/project-uuid/deploy') {
+                throw serverError;
+            }
+            return null;
+        });
+        await expect((0, deploy_1.deploy)([compiledExplore], {
+            ...deployOptions,
+            batchedDeploy: false,
+            dbtModelNames: ['orders'],
+        })).rejects.toBe(serverError);
+        expect(vi
+            .mocked(apiClient_1.lightdashApi)
+            .mock.calls.filter(([request]) => request.url ===
+            '/api/v2/projects/project-uuid/deploy')).toHaveLength(1);
+    });
     test('uses batched deploy by default and includes the completeness assertion in every batch body', async () => {
         vi.mocked(apiClient_1.lightdashApi).mockImplementation(async ({ url }) => {
             if (url.endsWith('/deploy')) {
@@ -107,6 +177,7 @@ describe('deploy completeness transport', () => {
         await (0, deploy_1.deploy)([compiledExplore], {
             ...deployOptions,
             complete: false,
+            dbtModelNames: ['orders', 'customers'],
         });
         expect(apiClient_1.lightdashApi).toHaveBeenCalledWith({
             method: 'POST',
@@ -117,6 +188,11 @@ describe('deploy completeness transport', () => {
                 complete: false,
             }),
         });
+        expect(apiClient_1.lightdashApi).toHaveBeenCalledWith({
+            method: 'POST',
+            url: '/api/v2/projects/project-uuid/deploy/deploy-session-uuid/finalize',
+            body: JSON.stringify({ dbtModelNames: ['orders', 'customers'] }),
+        });
     });
     test('falls back to the legacy deploy when session creation returns 404', async () => {
         vi.mocked(apiClient_1.lightdashApi).mockImplementation(async ({ url }) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/cli",
-  "version": "2.205.1",
+  "version": "2.206.0",
   "description": "Lightdash CLI tool",
   "license": "MIT",
   "repository": {
@@ -22,8 +22,8 @@
     "@actions/core": "1.11.1",
     "@aws-sdk/credential-providers": "3.995.0",
     "@casl/ability": "6.8.0",
-    "@lightdash/common": "2.205.1",
-    "@lightdash/warehouses": "2.205.1",
+    "@lightdash/common": "2.206.0",
+    "@lightdash/warehouses": "2.206.0",
     "@sidvind/better-ajv-errors": "7.0.0",
     "@types/columnify": "1.5.1",
     "ajv": "8.18.0",