@narrative.io/data-collaboration-sdk-ts 2.103.0 → 2.106.0

package/build/collaboration-policy/core/policy-validator.js

@@ -1,185 +0,0 @@
-import { visitFilterAttributes } from "./filter-utils";
-export function validatePolicyAgainstDataset(policy, _dataset, attributes) {
-    const result = {
-        isValid: true,
-        missingAttributes: [],
-        missingMappings: [],
-        errors: [],
-    };
-    const attributeByName = new Map();
-    for (const attribute of attributes) {
-        attributeByName.set(attribute.name, attribute);
-    }
-    // Validate the policy structure considering logical operators
-    const validation = validatePolicyStructure(policy.policy.definition.structure, attributeByName);
-    if (!validation.isValid) {
-        result.isValid = false;
-        result.errors.push(...validation.errors);
-        // Extract unique missing attributes for backwards compatibility
-        const allMissingAttributes = new Set();
-        for (const error of validation.errors) {
-            if (error.includes("Missing attributes:")) {
-                const attrs = error
-                    .split("Missing attributes:")[1]
-                    ?.split(",")
-                    .map((s) => s.trim()) || [];
-                for (const attr of attrs) {
-                    allMissingAttributes.add(attr);
-                }
-            }
-        }
-        result.missingAttributes = Array.from(allMissingAttributes);
-    }
-    const missingFilterAttributes = new Set();
-    const filters = policy.policy.definition.filters ?? [];
-    for (const filter of filters) {
-        collectMissingAttributesFromFilter(filter, attributeByName, missingFilterAttributes);
-    }
-    visitStructureFilters(policy.policy.definition.structure, (filter) => collectMissingAttributesFromFilter(filter, attributeByName, missingFilterAttributes), attributeByName);
-    if (missingFilterAttributes.size > 0) {
-        result.isValid = false;
-        const missingList = Array.from(missingFilterAttributes).sort();
-        result.errors.push(`Filters reference missing attributes: ${missingList.join(", ")}`);
-        for (const attr of missingList) {
-            if (!result.missingAttributes.includes(attr)) {
-                result.missingAttributes.push(attr);
-            }
-        }
-    }
-    return result;
-}
-function validatePolicyStructure(structure, attributeByName) {
-    if (!structure || typeof structure !== "object") {
-        return { isValid: true, errors: [] };
-    }
-    const struct = structure;
-    // Handle anyOf - at least one must be valid
-    if (struct.anyOf && Array.isArray(struct.anyOf)) {
-        for (const option of struct.anyOf) {
-            const optionValidation = validatePolicyStructure(option, attributeByName);
-            if (optionValidation.isValid) {
-                // If any option is valid, the whole anyOf is valid
-                return { isValid: true, errors: [] };
-            }
-        }
-        // If no options are valid, collect all errors
-        const allErrors = [];
-        for (const option of struct.anyOf) {
-            const optionValidation = validatePolicyStructure(option, attributeByName);
-            allErrors.push(...optionValidation.errors);
-        }
-        return {
-            isValid: false,
-            errors: [
-                `None of the anyOf options are satisfied. Issues: ${allErrors.join("; ")}`,
-            ],
-        };
-    }
-    // Handle allOf - all must be valid
-    if (struct.allOf && Array.isArray(struct.allOf)) {
-        const allErrors = [];
-        for (const requirement of struct.allOf) {
-            const reqValidation = validatePolicyStructure(requirement, attributeByName);
-            if (!reqValidation.isValid) {
-                allErrors.push(...reqValidation.errors);
-            }
-        }
-        return allErrors.length > 0
-            ? { isValid: false, errors: allErrors }
-            : { isValid: true, errors: [] };
-    }
-    // Handle single field with attribute
-    if (struct.field &&
-        struct.field.type ===
-            "attribute") {
-        const attributeName = struct.field
-            .attribute_name;
-        const errors = [];
-        // Check if attribute exists - we don't need to check mappings anymore
-        // since we use company_data.<attribute_name> format
-        if (!attributeByName.has(attributeName)) {
-            errors.push(`Missing attributes: ${attributeName}`);
-        }
-        return errors.length > 0
-            ? { isValid: false, errors }
-            : { isValid: true, errors: [] };
-    }
-    // For other structures, recursively validate
-    for (const value of Object.values(struct)) {
-        if (typeof value === "object") {
-            const childValidation = validatePolicyStructure(value, attributeByName);
-            if (!childValidation.isValid) {
-                return childValidation;
-            }
-        }
-    }
-    return { isValid: true, errors: [] };
-}
-/**
- * Checks if a branch is satisfied (all required attributes exist).
- * For anyOf: at least one child must be satisfied.
- * For allOf: all children must be satisfied.
- * For field nodes: the attribute must exist.
- */
-function isBranchSatisfied(node, attributeByName) {
-    if (!node || typeof node !== "object") {
-        return true;
-    }
-    if ("field" in node) {
-        const field = node.field;
-        if (field.type === "attribute" && field.attribute_name) {
-            return attributeByName.has(field.attribute_name);
-        }
-        return true;
-    }
-    if ("anyOf" in node && Array.isArray(node.anyOf)) {
-        return node.anyOf.some((child) => isBranchSatisfied(child, attributeByName));
-    }
-    if ("allOf" in node && Array.isArray(node.allOf)) {
-        return node.allOf.every((child) => isBranchSatisfied(child, attributeByName));
-    }
-    return true;
-}
-/**
- * Visits filters only in satisfied branches of the policy structure.
- * For anyOf nodes, only visits filters in branches where the required attributes exist.
- * For allOf nodes, visits filters in all branches (since all must be satisfied).
- */
-function visitStructureFilters(node, visitor, attributeByName) {
-    if (!node || typeof node !== "object") {
-        return;
-    }
-    if ("field" in node) {
-        // Only visit filters if this branch is satisfied
-        if (isBranchSatisfied(node, attributeByName)) {
-            const filters = node.filters;
-            if (filters) {
-                for (const filter of filters) {
-                    visitor(filter);
-                }
-            }
-        }
-        return;
-    }
-    if ("anyOf" in node && Array.isArray(node.anyOf)) {
-        // Only visit filters in satisfied branches
-        for (const child of node.anyOf) {
-            if (isBranchSatisfied(child, attributeByName)) {
-                visitStructureFilters(child, visitor, attributeByName);
-            }
-        }
-    }
-    if ("allOf" in node && Array.isArray(node.allOf)) {
-        // For allOf, visit all branches (they all must be satisfied)
-        for (const child of node.allOf) {
-            visitStructureFilters(child, visitor, attributeByName);
-        }
-    }
-}
-function collectMissingAttributesFromFilter(filter, attributesByName, missingAttributes) {
-    visitFilterAttributes(filter, (attributeName) => {
-        if (!attributesByName.has(attributeName)) {
-            missingAttributes.add(attributeName);
-        }
-    });
-}

package/build/collaboration-policy/core/sql-builder.d.ts

@@ -1,37 +0,0 @@
-import type { Attribute } from "../../attributes/types";
-import type { Dataset } from "../../datasets/types";
-import type { CollaborationPolicyType } from "../types";
-import type { PolicyMatchResult, PolicySqlFragments } from "./types";
-export declare function buildPolicySqlWithValidation(dataset: Dataset, policies: CollaborationPolicyType[], attributes: Attribute[]): PolicyMatchResult;
-/**
- * Categorizes collaboration policies into matching and non-matching groups based on dataset compatibility.
- *
- * This function validates each policy against the provided dataset and attributes to determine
- * whether the policy can be successfully applied. Policies are categorized as matching if they
- * have all required attributes available, or non-matching if they're missing required attributes
- * or mappings.
- *
- * @param policies - Array of collaboration policies to categorize
- * @param dataset - The dataset to validate policies against
- * @param attributes - Available attributes for validation
- * @returns Object containing matching policies, non-matching policies, and detailed skip information
- *
- * @example
- * ```typescript
- * const { matching, nonMatching, skippedDetails } = categorizePolicies(
- *   allPolicies,
- *   myDataset,
- *   availableAttributes
- * );
- *
- * console.log(`${matching.length} policies can be applied`);
- * console.log(`${nonMatching.length} policies were skipped`);
- * skippedDetails.forEach(skip => console.log(`Skipped ${skip.name}: ${skip.reason}`));
- * ```
- */
-export declare function categorizePolicies(policies: CollaborationPolicyType[], dataset: Dataset, attributes: Attribute[]): {
-    matching: CollaborationPolicyType[];
-    nonMatching: CollaborationPolicyType[];
-    skippedDetails: PolicyMatchResult["skippedPolicies"];
-};
-export declare function buildPolicySql(dataset: Dataset, policies: CollaborationPolicyType[], attributes: Attribute[]): PolicySqlFragments;

package/build/collaboration-policy/core/sql-builder.js

@@ -1,346 +0,0 @@
-import { normalizePathValue } from "../utils/path-helpers";
-import { buildSelectAlias, resolveAttributeExpression, } from "../utils/sql-helpers";
-import { filterToSql } from "./filter-builder";
-import { traversePolicy } from "./policy-traverser";
-import { validatePolicyAgainstDataset } from "./policy-validator";
-function buildAttributeIndexes(attributes) {
-    const byId = new Map();
-    const byName = new Map();
-    for (const attribute of attributes) {
-        byId.set(attribute.id, attribute);
-        byName.set(attribute.name, attribute);
-    }
-    return { byId, byName };
-}
-/**
- * Converts an ISO 8601 duration string to milliseconds for comparison.
- * This is an approximation for human-readable durations (months/years vary in length).
- *
- * @param duration - ISO 8601 duration string (e.g., "PT1H", "P1D", "P1M")
- * @returns Duration in milliseconds
- */
-function parseDurationToMilliseconds(duration) {
-    const regex = /^P(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/;
-    const matches = duration.match(regex);
-    if (!matches) {
-        throw new Error(`Invalid ISO 8601 duration format: ${duration}`);
-    }
-    const [, years, months, weeks, days, hours, minutes, seconds] = matches.map((m) => Number.parseInt(m || "0", 10));
-    // Approximate conversions (months = 30 days, years = 365 days)
-    const milliseconds = years * 365 * 24 * 60 * 60 * 1000 +
-        months * 30 * 24 * 60 * 60 * 1000 +
-        weeks * 7 * 24 * 60 * 60 * 1000 +
-        days * 24 * 60 * 60 * 1000 +
-        hours * 60 * 60 * 1000 +
-        minutes * 60 * 1000 +
-        seconds * 1000;
-    return milliseconds;
-}
-/**
- * Finds the smallest (most frequent) refresh schedule from a list of ISO 8601 duration strings.
- * The smallest duration means the most frequent refresh rate.
- *
- * @param schedules - Array of ISO 8601 duration strings
- * @returns The smallest duration in milliseconds, or 2592000000 (30 days/1 month) if no schedules provided
- */
-function getSmallestRefreshSchedule(schedules) {
-    // Default to monthly (30 days in milliseconds)
-    const defaultMonthlyMs = 30 * 24 * 60 * 60 * 1000;
-    if (schedules.length === 0) {
-        return defaultMonthlyMs;
-    }
-    if (schedules.length === 1) {
-        try {
-            return parseDurationToMilliseconds(schedules[0]);
-        }
-        catch (error) {
-            console.warn(`Invalid refresh schedule: ${schedules[0]}, using default`, error);
-            return defaultMonthlyMs;
-        }
-    }
-    // Find the schedule with the smallest duration (most frequent refresh)
-    let smallestValue = Number.MAX_SAFE_INTEGER;
-    for (const schedule of schedules) {
-        try {
-            const value = parseDurationToMilliseconds(schedule);
-            if (value < smallestValue) {
-                smallestValue = value;
-            }
-        }
-        catch (error) {
-            // If parsing fails, skip this schedule
-            console.warn(`Skipping invalid refresh schedule: ${schedule}`, error);
-        }
-    }
-    // If all schedules failed to parse, return default
-    return smallestValue === Number.MAX_SAFE_INTEGER
-        ? defaultMonthlyMs
-        : smallestValue;
-}
-/**
- * Builds structured WHERE clauses for a single branch node
- */
-function buildBranchWhere(node, attributeByName, datasetName) {
-    if (!node || typeof node !== "object") {
-        return null;
-    }
-    // Handle field nodes (leaf nodes with attributes)
-    if ("field" in node) {
-        const extendedNode = node;
-        const field = extendedNode.field;
-        const attributeName = field.attribute_name;
-        const attribute = attributeByName.get(attributeName);
-        if (!attribute) {
-            return null;
-        }
-        const clauses = [];
-        // Add IS NOT NULL filters for additional_required_properties
-        if (field.additional_required_properties) {
-            for (const path of field.additional_required_properties) {
-                const pathStr = normalizePathValue(path);
-                const expression = `company_data."${datasetName}"."_rosetta_stone"."${attribute.name}"."${pathStr}"`;
-                clauses.push(`(${expression} IS NOT NULL)`);
-            }
-        }
-        // Add filters
-        if (extendedNode.filters) {
-            for (const filter of extendedNode.filters) {
-                const sql = filterToSql(filter, attributeByName, datasetName);
-                clauses.push(sql);
-            }
-        }
-        // If only one clause, return as string
-        if (clauses.length === 0) {
-            return null;
-        }
-        if (clauses.length === 1) {
-            return clauses[0];
-        }
-        // Multiple clauses for a single field - AND them together
-        return {
-            operation: "AND",
-            fragments: clauses,
-        };
-    }
-    // Handle anyOf nodes
-    if ("anyOf" in node && Array.isArray(node.anyOf)) {
-        const satisfiedBranches = [];
-        for (const child of node.anyOf) {
-            if (isBranchSatisfied(child, attributeByName)) {
-                const branchWhere = buildBranchWhere(child, attributeByName, datasetName);
-                if (branchWhere) {
-                    satisfiedBranches.push(branchWhere);
-                }
-            }
-        }
-        if (satisfiedBranches.length === 0) {
-            return null;
-        }
-        if (satisfiedBranches.length === 1) {
-            return satisfiedBranches[0];
-        }
-        // Multiple satisfied anyOf branches - OR them together
-        return {
-            operation: "OR",
-            fragments: satisfiedBranches,
-        };
-    }
-    // Handle allOf nodes
-    if ("allOf" in node && Array.isArray(node.allOf)) {
-        const allBranches = [];
-        for (const child of node.allOf) {
-            const branchWhere = buildBranchWhere(child, attributeByName, datasetName);
-            if (branchWhere) {
-                allBranches.push(branchWhere);
-            }
-        }
-        if (allBranches.length === 0) {
-            return null;
-        }
-        if (allBranches.length === 1) {
-            return allBranches[0];
-        }
-        // Multiple allOf branches - AND them together
-        return {
-            operation: "AND",
-            fragments: allBranches,
-        };
-    }
-    return null;
-}
-/**
- * Checks if a branch is satisfied (all required attributes exist)
- */
-function isBranchSatisfied(node, attributeByName) {
-    if (!node || typeof node !== "object") {
-        return true;
-    }
-    if ("field" in node) {
-        const field = node
-            .field;
-        if (field.type === "attribute") {
-            const attributeName = field.attribute_name;
-            return attributeByName.has(attributeName);
-        }
-        return true;
-    }
-    if ("anyOf" in node && Array.isArray(node.anyOf)) {
-        return node.anyOf.some((child) => isBranchSatisfied(child, attributeByName));
-    }
-    if ("allOf" in node && Array.isArray(node.allOf)) {
-        return node.allOf.every((child) => isBranchSatisfied(child, attributeByName));
-    }
-    return true;
-}
-function buildPolicySqlUsingIndexes(policies, attributeByName, datasetName) {
-    const globalRequiredAttributePaths = new Map();
-    const policyWhereGroups = [];
-    for (const policy of policies) {
-        const policyRequiredAttributePaths = new Map();
-        const policyWhereClausesSet = new Set();
-        // Still use traversePolicy to collect required attributes for SELECT
-        traversePolicy(policy, attributeByName, policyRequiredAttributePaths, policyWhereClausesSet, datasetName);
-        // Build structured WHERE for this policy
-        const policyWhere = buildBranchWhere(policy.policy.definition.structure, attributeByName, datasetName);
-        if (policyWhere) {
-            policyWhereGroups.push(policyWhere);
-        }
-        for (const [attributeName, paths] of policyRequiredAttributePaths) {
-            const globalPaths = globalRequiredAttributePaths.get(attributeName) ?? new Set();
-            for (const path of paths) {
-                globalPaths.add(path);
-            }
-            globalRequiredAttributePaths.set(attributeName, globalPaths);
-        }
-    }
-    const selectClauses = new Set();
-    for (const [attributeName, _paths] of globalRequiredAttributePaths) {
-        const attribute = attributeByName.get(attributeName);
-        if (!attribute) {
-            continue;
-        }
-        // Always select the root attribute (without path)
-        const expression = resolveAttributeExpression(attribute, "", // Empty path for root attribute
-        datasetName);
-        const alias = buildSelectAlias(attributeName, "");
-        selectClauses.add(`${expression} AS ${alias}`);
-    }
-    // Build the root AND structure
-    // Root level is always AND (combines policies)
-    // Each policy's branches are already OR'd/AND'd by buildBranchWhere
-    let where;
-    if (policyWhereGroups.length === 0) {
-        where = [];
-    }
-    else {
-        // Wrap all policies in root AND
-        // Each policy in policyWhereGroups is already a complete policy structure
-        where = [
-            {
-                fragments: policyWhereGroups,
-                operation: "AND",
-            },
-        ];
-    }
-    return {
-        select: Array.from(selectClauses),
-        where,
-    };
-}
-export function buildPolicySqlWithValidation(dataset, policies, attributes) {
-    const attributeIndexes = buildAttributeIndexes(attributes);
-    const validPolicies = [];
-    const skippedPolicies = [];
-    const warnings = [];
-    // Validate each policy against the dataset
-    for (const policy of policies) {
-        const validation = validatePolicyAgainstDataset(policy, dataset, attributes);
-        if (validation.isValid) {
-            validPolicies.push(policy);
-        }
-        else {
-            skippedPolicies.push({
-                name: policy.display_name || policy.name,
-                reason: "Missing required attributes or mappings",
-                details: validation.errors,
-            });
-        }
-    }
-    // Check if we have at least one valid policy
-    if (validPolicies.length === 0) {
-        const errorMessage = policies.length === 0
-            ? "No policies provided"
-            : `No policies are compatible with dataset "${dataset.display_name || dataset.name}".`;
-        const details = skippedPolicies.length > 0
-            ? [`Skipped policies: ${skippedPolicies.map((p) => p.name).join(", ")}`]
-            : [];
-        throw new Error(`${errorMessage}\n${details.join("\n")}`);
-    }
-    if (skippedPolicies.length > 0) {
-        warnings.push(`Skipped ${skippedPolicies.length} incompatible policies: ${skippedPolicies.map((p) => p.name).join(", ")}`);
-    }
-    // Build SQL for valid policies
-    const sqlFragments = buildPolicySqlUsingIndexes(validPolicies, attributeIndexes.byName, dataset.name);
-    // Calculate the smallest refresh schedule from applied policies
-    const refreshSchedules = validPolicies
-        .map((p) => p.policy.metadata?.refresh_schedule?.max)
-        .filter((schedule) => schedule !== undefined);
-    const refreshSchedule = getSmallestRefreshSchedule(refreshSchedules);
-    return {
-        ...sqlFragments,
-        appliedPolicies: validPolicies.map((p) => p.name),
-        skippedPolicies,
-        warnings,
-        refresh_schedule: refreshSchedule,
-    };
-}
-/**
- * Categorizes collaboration policies into matching and non-matching groups based on dataset compatibility.
- *
- * This function validates each policy against the provided dataset and attributes to determine
- * whether the policy can be successfully applied. Policies are categorized as matching if they
- * have all required attributes available, or non-matching if they're missing required attributes
- * or mappings.
- *
- * @param policies - Array of collaboration policies to categorize
- * @param dataset - The dataset to validate policies against
- * @param attributes - Available attributes for validation
- * @returns Object containing matching policies, non-matching policies, and detailed skip information
- *
- * @example
- * ```typescript
- * const { matching, nonMatching, skippedDetails } = categorizePolicies(
- *   allPolicies,
- *   myDataset,
- *   availableAttributes
- * );
- *
- * console.log(`${matching.length} policies can be applied`);
- * console.log(`${nonMatching.length} policies were skipped`);
- * skippedDetails.forEach(skip => console.log(`Skipped ${skip.name}: ${skip.reason}`));
- * ```
- */
-export function categorizePolicies(policies, dataset, attributes) {
-    const matching = [];
-    const nonMatching = [];
-    const skippedDetails = [];
-    for (const policy of policies) {
-        const validation = validatePolicyAgainstDataset(policy, dataset, attributes);
-        if (validation.isValid) {
-            matching.push(policy);
-        }
-        else {
-            nonMatching.push(policy);
-            skippedDetails.push({
-                name: policy.display_name || policy.name,
-                reason: "Missing required attributes or mappings",
-                details: validation.errors,
-            });
-        }
-    }
-    return { matching, nonMatching, skippedDetails };
-}
-export function buildPolicySql(dataset, policies, attributes) {
-    const attributeIndexes = buildAttributeIndexes(attributes);
-    return buildPolicySqlUsingIndexes(policies, attributeIndexes.byName, dataset.name);
-}

package/build/collaboration-policy/scripts/generate-policy-schema.d.ts

@@ -1,2 +0,0 @@
-#!/usr/bin/env bun
-export {};

package/build/collaboration-policy/scripts/generate-policy-schema.js

@@ -1,4 +0,0 @@
-#!/usr/bin/env bun
-import { buildCollaborationPolicyJsonSchema } from "../types";
-const schema = buildCollaborationPolicyJsonSchema();
-console.log(JSON.stringify(schema, null, 2));