@owox/backend 0.35.0 → 0.37.0

package/dist/src/data-marts/services/connector/run-log-flusher.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunLogFlusher = void 0;
+exports.createRunLogSnapshotReader = createRunLogSnapshotReader;
+function createRunLogSnapshotReader(logs, errors) {
+    const serializedLogs = [];
+    const serializedErrors = [];
+    const extend = (cache, buffer) => {
+        for (let i = cache.length; i < buffer.length; i++) {
+            cache.push(JSON.stringify(buffer[i]));
+        }
+        return cache.slice();
+    };
+    return () => ({
+        logs: extend(serializedLogs, logs),
+        errors: extend(serializedErrors, errors),
+    });
+}
+class RunLogFlusher {
+    intervalMs;
+    getSnapshot;
+    write;
+    onError;
+    timer = null;
+    lastCount = -1;
+    inFlight = Promise.resolve();
+    persisted = null;
+    constructor(intervalMs, getSnapshot, write, onError) {
+        this.intervalMs = intervalMs;
+        this.getSnapshot = getSnapshot;
+        this.write = write;
+        this.onError = onError;
+    }
+    start() {
+        if (this.intervalMs <= 0 || this.timer !== null)
+            return;
+        this.timer = setInterval(() => {
+            this.inFlight = this.inFlight.then(() => this.flushIfDirty());
+        }, this.intervalMs);
+    }
+    async flushIfDirty() {
+        const snapshot = this.getSnapshot();
+        const count = snapshot.logs.length + snapshot.errors.length;
+        if (count === 0 || count === this.lastCount)
+            return;
+        try {
+            await this.write(snapshot);
+            this.lastCount = count;
+            this.persisted = snapshot;
+        }
+        catch (error) {
+            this.onError?.(error);
+        }
+    }
+    persistedSnapshot() {
+        return this.persisted;
+    }
+    async stop() {
+        if (this.timer !== null) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+        await this.inFlight;
+    }
+}
+exports.RunLogFlusher = RunLogFlusher;
+//# sourceMappingURL=run-log-flusher.js.map

package/dist/src/data-marts/services/context/context-access.service.js

@@ -56,7 +56,7 @@ let ContextAccessService = class ContextAccessService {
             const ownerStatus = await this.accessDecisionService.getOwnerStatus(userId, access_decision_types_1.EntityType.DATA_MART, dataMartId);
             const isTechOwnerWithEditorRole = ownerStatus === access_decision_types_1.OwnerStatus.TECH_OWNER && roles.includes('editor');
             if (!isTechOwnerWithEditorRole) {
-                throw new common_1.ForbiddenException('Only Data Mart Technical Owners with Technical User role or Project Admins can manage Data Mart contexts');
+                throw new common_1.ForbiddenException('Only Data Owners of the Data Mart who have the Data Owner role, or Project Admins, can manage Data Mart contexts');
             }
         }
         await this.contextService.validateContextIds(contextIds, projectId);
@@ -71,7 +71,7 @@ let ContextAccessService = class ContextAccessService {
             const ownerStatus = await this.accessDecisionService.getOwnerStatus(userId, access_decision_types_1.EntityType.STORAGE, storageId);
             const isOwnerWithEditorRole = ownerStatus === access_decision_types_1.OwnerStatus.OWNER && roles.includes('editor');
             if (!isOwnerWithEditorRole) {
-                throw new common_1.ForbiddenException('Only Data Storage Owners with Technical User role or Project Admins can manage Data Storage contexts');
+                throw new common_1.ForbiddenException('Only Data Storage Owners who have the Data Owner role, or Project Admins, can manage Data Storage contexts');
             }
         }
         await this.contextService.validateContextIds(contextIds, projectId);

package/dist/src/data-marts/services/data-mart.service.js

@@ -88,6 +88,7 @@ let DataMartService = DataMartService_1 = class DataMartService {
             'dm.title',
             'dm.status',
             'dm.description',
+            'dm.icon',
             'dm.definitionType',
             'dm.createdById',
             'dm.createdAt',
@@ -183,6 +184,7 @@ let DataMartService = DataMartService_1 = class DataMartService {
             'dm.title',
             'dm.status',
             'dm.description',
+            'dm.icon',
             'dm.schema',
             'dm.dataLastUpdated',
         ])

package/dist/src/data-marts/services/member-ownership-warnings.service.js

@@ -42,7 +42,7 @@ let MemberOwnershipWarningsService = MemberOwnershipWarningsService_1 = class Me
             .getRawMany();
         return techOwners.map(({ userId }) => ({
             userId,
-            warning: 'Technical Owner — requires Technical User role to be effective',
+            warning: 'Data Owner — requires the Data Owner role to be effective',
         }));
     }
 };

package/dist/src/data-marts/services/report-access.service.js

@@ -30,7 +30,7 @@ const EMPTY_CAPABILITIES = Object.freeze({
 const MUTATE_DENIED_MESSAGES = {
     'not-owner': 'You are not an owner of this report. Only report owners can modify it.',
     'not-found': 'Report not found.',
-    ineffective: 'The destination for this report is not accessible to you. Ask a Technical User to share the destination or replace it.',
+    ineffective: 'The destination for this report is not accessible to you. Ask the destination owner or a Project Admin to share it, or replace the destination.',
     'dm-invisible': 'You do not have access to the DataMart for this report.',
 };
 const OPERATE_DENIED_MESSAGES = {

package/dist/src/data-marts/services/report-sql-composer.service.js

@@ -36,6 +36,20 @@ const field_aggregation_governance_1 = require("../dto/schemas/field-aggregation
 const aggregation_labels_1 = require("../dto/schemas/aggregation-labels");
 const field_type_category_1 = require("../dto/schemas/field-type-category");
 const report_data_header_dto_1 = require("../dto/domain/report-data-header.dto");
+function withMetricsShownForDryRun(dataMart, metricNames) {
+    const fields = dataMart.schema?.fields ?? [];
+    const isHiddenMetric = (field) => field.isHiddenForReporting === true &&
+        (0, calculated_field_utils_1.isCalculatedField)(field) &&
+        metricNames.includes(field.name);
+    if (!fields.some(isHiddenMetric))
+        return dataMart;
+    return Object.assign(Object.create(Object.getPrototypeOf(dataMart)), dataMart, {
+        schema: {
+            ...dataMart.schema,
+            fields: fields.map(field => isHiddenMetric(field) ? { ...field, isHiddenForReporting: false } : field),
+        },
+    });
+}
 let ReportSqlComposerService = class ReportSqlComposerService {
     blendedReportDataService;
     queryBuilderFacade;
@@ -348,7 +362,7 @@ let ReportSqlComposerService = class ReportSqlComposerService {
     }
     async composeMetricsOnly(dataMart, metricNames, accessor, precomputedBlendableSchema, tableReferences) {
         const plan = {
-            dataMart,
+            dataMart: withMetricsShownForDryRun(dataMart, metricNames),
             columnConfig: metricNames,
         };
         const joinedMetrics = (0, calculated_field_utils_1.calculatedFieldsOf)(dataMart.schema?.fields ?? []).filter(f => metricNames.includes(f.name) && (0, formula_live_reference_1.hasLiveJoinedReference)(f.calculated.formula));

package/dist/src/data-marts/services/report.service.d.ts

@@ -28,5 +28,6 @@ export declare class ReportService {
     updateReportWithVersionControl(report: Report): Promise<void>;
     deleteReport(report: Report): Promise<void>;
     countByDataMartIdAndProjectId(dataMartId: string, projectId: string): Promise<number>;
+    countByDataMartIds(dataMartIds: string[]): Promise<Map<string, number>>;
     deleteAllByDataMartIdAndProjectId(dataMartId: string, projectId: string): Promise<void>;
 }

package/dist/src/data-marts/services/report.service.js

@@ -198,6 +198,19 @@ let ReportService = class ReportService {
             },
         });
     }
+    async countByDataMartIds(dataMartIds) {
+        if (dataMartIds.length === 0)
+            return new Map();
+        const rows = await this.repository
+            .createQueryBuilder('r')
+            .leftJoin('r.dataMart', 'dm')
+            .where('dm.id IN (:...ids)', { ids: dataMartIds })
+            .select('dm.id', 'dataMartId')
+            .addSelect('COUNT(*)', 'count')
+            .groupBy('dm.id')
+            .getRawMany();
+        return new Map(rows.map(row => [row.dataMartId, Number(row.count)]));
+    }
     async deleteAllByDataMartIdAndProjectId(dataMartId, projectId) {
         const reports = await this.repository.find({
             where: {

package/dist/src/data-marts/services/scheduled-trigger.service.d.ts

@@ -16,6 +16,7 @@ export declare class ScheduledTriggerService {
     constructor(triggerRepository: Repository<DataMartScheduledTrigger>);
     getByIdAndDataMartIdAndProjectId(id: string, dataMartId: string, projectId: string): Promise<DataMartScheduledTrigger>;
     getByIdAndProjectId(id: string, projectId: string): Promise<DataMartScheduledTrigger>;
+    countByDataMartIds(dataMartIds: string[]): Promise<Map<string, number>>;
     getAllByDataMartIdAndProjectId(dataMartId: string, projectId: string): Promise<DataMartScheduledTrigger[]>;
     listVisibleByProject(options: ListVisibleProjectScheduledTriggersOptions): Promise<DataMartScheduledTrigger[]>;
     deleteAllByDataMartIdAndProjectId(dataMartId: string, projectId: string): Promise<void>;

package/dist/src/data-marts/services/scheduled-trigger.service.js

@@ -50,6 +50,19 @@ let ScheduledTriggerService = class ScheduledTriggerService {
         }
         return trigger;
     }
+    async countByDataMartIds(dataMartIds) {
+        if (dataMartIds.length === 0)
+            return new Map();
+        const rows = await this.triggerRepository
+            .createQueryBuilder('t')
+            .leftJoin('t.dataMart', 'dm')
+            .where('dm.id IN (:...ids)', { ids: dataMartIds })
+            .select('dm.id', 'dataMartId')
+            .addSelect('COUNT(*)', 'count')
+            .groupBy('dm.id')
+            .getRawMany();
+        return new Map(rows.map(row => [row.dataMartId, Number(row.count)]));
+    }
     async getAllByDataMartIdAndProjectId(dataMartId, projectId) {
         return this.triggerRepository.find({
             where: {

package/dist/src/data-marts/use-cases/create-data-mart.service.js

@@ -76,6 +76,7 @@ let CreateDataMartService = CreateDataMartService_1 = class CreateDataMartServic
         }
         const dataMart = this.dataMartService.create({
             title: command.title,
+            icon: command.icon,
             projectId: command.projectId,
             createdById: command.userId,
             storage: dataStorage,

package/dist/src/data-marts/use-cases/create-report.service.js

@@ -37,6 +37,7 @@ const access_decision_1 = require("../services/access-decision");
 const output_controls_validator_service_1 = require("../services/output-controls-validator.service");
 const report_access_service_1 = require("../services/report-access.service");
 const unique_count_sources_1 = require("../dto/schemas/unique-count-sources");
+const auto_aggregation_opt_out_schema_1 = require("../dto/schemas/auto-aggregation-opt-out.schema");
 const advanced_search_index_sync_service_1 = require("../services/advanced-search-index-sync.service");
 const search_facade_1 = require("../../common/search/search.facade");
 const looker_studio_report_service_1 = require("../services/looker-studio-report.service");
@@ -118,6 +119,7 @@ let CreateReportService = class CreateReportService {
             aggregationConfig: command.aggregationConfig ?? null,
             dateTruncConfig: command.dateTruncConfig ?? null,
             uniqueCountConfig: (0, unique_count_sources_1.foldEmptyUniqueCountConfig)(command.uniqueCountConfig),
+            autoAggregationOptOut: (0, auto_aggregation_opt_out_schema_1.selectedAutoAggregationOptOut)(command.autoAggregationOptOut, command.columnConfig),
         });
         const restoredReport = await this.lookerStudioReportService.restoreIfDeleted(report);
         const newReport = restoredReport ?? (await this.reportRepository.save(report));

package/dist/src/data-marts/use-cases/get-model-canvas-data-marts.service.d.ts

@@ -5,12 +5,16 @@ import { AccessDecisionService } from '../services/access-decision';
 import { ContextAccessService } from '../services/context/context-access.service';
 import { DataMartService } from '../services/data-mart.service';
 import { DataStorageService } from '../services/data-storage.service';
+import { ReportService } from '../services/report.service';
+import { ScheduledTriggerService } from '../services/scheduled-trigger.service';
 export declare class GetModelCanvasDataMartsService {
     private readonly dataMartService;
     private readonly dataStorageService;
     private readonly contextAccessService;
     private readonly mapper;
     private readonly accessDecisionService;
-    constructor(dataMartService: DataMartService, dataStorageService: DataStorageService, contextAccessService: ContextAccessService, mapper: ModelCanvasMapper, accessDecisionService: AccessDecisionService);
+    private readonly scheduledTriggerService;
+    private readonly reportService;
+    constructor(dataMartService: DataMartService, dataStorageService: DataStorageService, contextAccessService: ContextAccessService, mapper: ModelCanvasMapper, accessDecisionService: AccessDecisionService, scheduledTriggerService: ScheduledTriggerService, reportService: ReportService);
     run(command: GetModelCanvasDataMartsCommand): Promise<ModelCanvasDataMartsDto>;
 }

package/dist/src/data-marts/use-cases/get-model-canvas-data-marts.service.js

@@ -17,6 +17,8 @@ const access_decision_1 = require("../services/access-decision");
 const context_access_service_1 = require("../services/context/context-access.service");
 const data_mart_service_1 = require("../services/data-mart.service");
 const data_storage_service_1 = require("../services/data-storage.service");
+const report_service_1 = require("../services/report.service");
+const scheduled_trigger_service_1 = require("../services/scheduled-trigger.service");
 const CANVAS_DATA_MARTS_PAGE_SIZE = 1000;
 let GetModelCanvasDataMartsService = class GetModelCanvasDataMartsService {
     dataMartService;
@@ -24,12 +26,16 @@ let GetModelCanvasDataMartsService = class GetModelCanvasDataMartsService {
     contextAccessService;
     mapper;
     accessDecisionService;
-    constructor(dataMartService, dataStorageService, contextAccessService, mapper, accessDecisionService) {
+    scheduledTriggerService;
+    reportService;
+    constructor(dataMartService, dataStorageService, contextAccessService, mapper, accessDecisionService, scheduledTriggerService, reportService) {
         this.dataMartService = dataMartService;
         this.dataStorageService = dataStorageService;
         this.contextAccessService = contextAccessService;
         this.mapper = mapper;
         this.accessDecisionService = accessDecisionService;
+        this.scheduledTriggerService = scheduledTriggerService;
+        this.reportService = reportService;
     }
     async run(command) {
         if (!command.userId) {
@@ -52,7 +58,12 @@ let GetModelCanvasDataMartsService = class GetModelCanvasDataMartsService {
             limit: CANVAS_DATA_MARTS_PAGE_SIZE,
             offset,
         });
-        return this.mapper.toDataMartsDto(items, total, offset);
+        const ids = items.map(dataMart => dataMart.id);
+        const [triggerCounts, reportCounts] = await Promise.all([
+            this.scheduledTriggerService.countByDataMartIds(ids),
+            this.reportService.countByDataMartIds(ids),
+        ]);
+        return this.mapper.toDataMartsDto(items, total, offset, { triggerCounts, reportCounts });
     }
 };
 exports.GetModelCanvasDataMartsService = GetModelCanvasDataMartsService;
@@ -62,6 +73,8 @@ exports.GetModelCanvasDataMartsService = GetModelCanvasDataMartsService = __deco
         data_storage_service_1.DataStorageService,
         context_access_service_1.ContextAccessService,
         model_canvas_mapper_1.ModelCanvasMapper,
-        access_decision_1.AccessDecisionService])
+        access_decision_1.AccessDecisionService,
+        scheduled_trigger_service_1.ScheduledTriggerService,
+        report_service_1.ReportService])
 ], GetModelCanvasDataMartsService);
 //# sourceMappingURL=get-model-canvas-data-marts.service.js.map

package/dist/src/data-marts/use-cases/list-project-scheduled-triggers.service.js

@@ -121,7 +121,7 @@ let ListProjectScheduledTriggersService = class ListProjectScheduledTriggersServ
             if (trigger.type === scheduled_trigger_type_enum_1.ScheduledTriggerType.CONNECTOR_RUN &&
                 trigger.dataMart.definition &&
                 (0, data_mart_definition_guards_1.isConnectorDefinition)(trigger.dataMart.definition)) {
-                const maskedDefinition = await this.connectorSecretService.mask(trigger.dataMart.definition);
+                const maskedDefinition = await this.connectorSecretService.mask(projectId, trigger.dataMart.definition);
                 return this.cloneTriggerWithConfig(trigger, {
                     type: ScheduledConnectorRunConfigType,
                     connector: maskedDefinition,

package/dist/src/data-marts/use-cases/preview-data-mart.service.d.ts

@@ -0,0 +1,59 @@
+import { HttpException } from '@nestjs/common';
+import { TypeResolver } from '../../common/resolver/type-resolver';
+import type { Role as RoleType } from '@owox/idp-protocol';
+import { DataStorageType } from '../data-storage-types/enums/data-storage-type.enum';
+import { DataStorageErrorMapper } from '../data-storage-types/interfaces/data-storage-error-mapper.interface';
+import { DataStorageReportReader } from '../data-storage-types/interfaces/data-storage-report-reader.interface';
+import { AccessDecisionService } from '../services/access-decision';
+import { BlendableSchemaService } from '../services/blendable-schema.service';
+import { DataMartService } from '../services/data-mart.service';
+import { ProjectBillingService } from '../services/project-billing/project-billing.service';
+import { ReportSqlComposerService } from '../services/report-sql-composer.service';
+export declare const PREVIEW_DEFAULT_LIMIT = 10;
+export declare const PREVIEW_MAX_LIMIT = 1000;
+export declare const DEFAULT_PREVIEW_DEADLINE_MS = 150000;
+export declare class PreviewDataMartCommand {
+    readonly dataMartId: string;
+    readonly projectId: string;
+    readonly userId: string;
+    readonly roles: RoleType[];
+    readonly limit: number | undefined;
+    readonly filters: unknown;
+    readonly sort: unknown;
+    constructor(dataMartId: string, projectId: string, userId: string, roles: RoleType[], limit: number | undefined, filters: unknown, sort?: unknown);
+}
+export interface DataMartPreviewColumn {
+    name: string;
+    alias?: string;
+    type?: string;
+}
+export type PreviewCell = string | number | boolean | null;
+export interface DataMartPreviewResult {
+    columns: DataMartPreviewColumn[];
+    rows: PreviewCell[][];
+    rowCount: number;
+    limit: number;
+    truncated: boolean;
+}
+export declare const CLIENT_CLOSED_REQUEST_STATUS = 499;
+export declare class PreviewAbortedError extends HttpException {
+    constructor();
+}
+export declare class PreviewDataMartService {
+    private readonly dataMartService;
+    private readonly blendableSchemaService;
+    private readonly composer;
+    private readonly readerResolver;
+    private readonly accessDecisionService;
+    private readonly errorMapperResolver;
+    private readonly projectBillingService;
+    private readonly deadlineMs;
+    private readonly logger;
+    constructor(dataMartService: DataMartService, blendableSchemaService: BlendableSchemaService, composer: ReportSqlComposerService, readerResolver: TypeResolver<DataStorageType, DataStorageReportReader>, accessDecisionService: AccessDecisionService, errorMapperResolver: TypeResolver<DataStorageType, DataStorageErrorMapper>, projectBillingService: ProjectBillingService, deadlineMs?: number);
+    run(command: PreviewDataMartCommand, signal?: AbortSignal): Promise<DataMartPreviewResult>;
+    private parseLimit;
+    private parseSort;
+    private parseFilters;
+    private readRows;
+}
+export declare function toPreviewCell(value: unknown): PreviewCell;