@narrative.io/data-collaboration-sdk-ts 2.98.1-beta.0 → 2.99.0

package/build/collaboration-policy/core/jsonschema/sql-poc.js

@@ -1,305 +0,0 @@
-import { buildSelectAlias, resolveAttributeExpression, } from "../../utils/sql-helpers";
-import { filterToSql } from "../filter-builder";
-import { evaluateJsonSchemaPolicies } from "./policy-evaluator";
-/**
- * Convert an ISO 8601 duration string (e.g. "P1D", "PT1H") into milliseconds.
- * This is an approximation (months and years are treated as 30 and 365 days respectively).
- *
- * @param duration - The ISO 8601 duration string.
- * @returns The approximate duration in milliseconds.
- * @throws If the duration string does not match the ISO 8601 duration format.
- */
-function parseDurationToMilliseconds(duration) {
-    const regex = /^P(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/;
-    const matches = duration.match(regex);
-    if (!matches) {
-        throw new Error(`Invalid ISO 8601 duration format: ${duration}`);
-    }
-    const [, years, months, weeks, days, hours, minutes, seconds] = matches.map((m) => Number.parseInt(m || "0", 10));
-    const milliseconds = years * 365 * 24 * 60 * 60 * 1000 +
-        months * 30 * 24 * 60 * 60 * 1000 +
-        weeks * 7 * 24 * 60 * 60 * 1000 +
-        days * 24 * 60 * 60 * 1000 +
-        hours * 60 * 60 * 1000 +
-        minutes * 60 * 1000 +
-        seconds * 1000;
-    return milliseconds;
-}
-/**
- * Find the "smallest" (most frequent) ISO 8601 duration within a list.
- *
- * @param schedules - Array of ISO 8601 duration strings.
- * @returns The smallest duration as an ISO 8601 string, or "P1M" if none are valid.
- */
-function getSmallestRefreshSchedule(schedules) {
-    const defaultMonthly = "P1M";
-    if (schedules.length === 0)
-        return defaultMonthly;
-    if (schedules.length === 1) {
-        try {
-            parseDurationToMilliseconds(schedules[0]);
-            return schedules[0];
-        }
-        catch {
-            return defaultMonthly;
-        }
-    }
-    let smallestValue = Number.MAX_SAFE_INTEGER;
-    let smallestSchedule = null;
-    for (const schedule of schedules) {
-        try {
-            const value = parseDurationToMilliseconds(schedule);
-            if (value < smallestValue) {
-                smallestValue = value;
-                smallestSchedule = schedule;
-            }
-        }
-        catch {
-            // skip invalid schedule
-        }
-    }
-    return smallestSchedule ?? defaultMonthly;
-}
-/**
- * Build SQL fragments (SELECT + WHERE) from branch matches across all applied policies.
- *
- * SELECT: one entry per unique required attribute (root level, no subfield paths):
- *   company_data."<dataset>"."_rosetta_stone"."<attr>" AS "<attr>"
- *
- * WHERE: structured as root AND → per-policy structure → OR of branches.
- *   Each branch contains IS NOT NULL clauses (from additionalRequiredProperties)
- *   and filter SQL (from filters), ANDed together.
- *
- * @param branchMatches - Per-policy branch match results from evaluateJsonSchemaPolicies.
- * @param attributeByName - Map of attribute name → Attribute for SQL expression building.
- * @param datasetName - Dataset name used in SQL column paths.
- * @returns PolicySqlFragments with select and where arrays.
- */
-function buildSqlFromBranchMatches(branchMatches, attributeByName, datasetName) {
-    // --- SELECT ---
-    // Collect all unique required attributes across all policies and branches
-    const globalRequiredAttributes = new Set();
-    for (const policyMatch of branchMatches) {
-        for (const branch of policyMatch.branches) {
-            for (const attrName of branch.requiredAttributes) {
-                globalRequiredAttributes.add(attrName);
-            }
-        }
-    }
-    const selectClauses = new Set();
-    for (const attrName of globalRequiredAttributes) {
-        const attribute = attributeByName.get(attrName);
-        if (!attribute)
-            continue;
-        const expression = resolveAttributeExpression(attribute, "", datasetName);
-        const alias = buildSelectAlias(attrName, "");
-        selectClauses.add(`${expression} AS ${alias}`);
-    }
-    // --- WHERE ---
-    const policyWhereGroups = [];
-    for (const policyMatch of branchMatches) {
-        const branchWheres = [];
-        for (const branch of policyMatch.branches) {
-            const branchClauses = [];
-            // 1. IS NOT NULL clauses derived from $ref/$defs resolution
-            for (const req of branch.additionalRequiredProperties) {
-                const attribute = attributeByName.get(req.attributeName);
-                if (!attribute)
-                    continue;
-                for (const path of req.paths) {
-                    const expression = resolveAttributeExpression(attribute, path, datasetName);
-                    branchClauses.push(`(${expression} IS NOT NULL)`);
-                }
-            }
-            // 2. Filter SQL
-            for (const filter of branch.filters) {
-                const sql = filterToSql(filter, attributeByName, datasetName);
-                branchClauses.push(sql);
-            }
-            if (branchClauses.length === 0) {
-                continue; // branch has no WHERE contribution
-            }
-            if (branchClauses.length === 1) {
-                branchWheres.push(branchClauses[0]);
-            }
-            else {
-                branchWheres.push({
-                    operation: "AND",
-                    fragments: branchClauses,
-                });
-            }
-        }
-        if (branchWheres.length === 0) {
-            continue; // policy has no WHERE contribution
-        }
-        if (branchWheres.length === 1) {
-            policyWhereGroups.push(branchWheres[0]);
-        }
-        else {
-            // Multiple branches → OR them (anyOf semantics)
-            policyWhereGroups.push({
-                operation: "OR",
-                fragments: branchWheres,
-            });
-        }
-    }
-    let where;
-    if (policyWhereGroups.length === 0) {
-        where = [];
-    }
-    else {
-        // Root AND wrapping all policies
-        where = [
-            {
-                fragments: policyWhereGroups,
-                operation: "AND",
-            },
-        ];
-    }
-    return {
-        select: Array.from(selectClauses),
-        where,
-    };
-}
-/**
- * Categorize JSON-Schema-based connector policies into matching and non-matching
- * groups based on dataset compatibility.
- *
- * @param policies - JSON-Schema-based connector policies.
- * @param dataset - Dataset whose schema is used for validation.
- * @param attributes - Attributes associated with the dataset.
- * @returns Object with matching policies, nonMatching policies, and skip details.
- */
-export function categorizePolicies(policies, dataset, attributes) {
-    const evaluations = evaluateJsonSchemaPolicies(dataset, attributes, policies);
-    const matching = [];
-    const nonMatching = [];
-    const skippedDetails = [];
-    for (const evalResult of evaluations) {
-        const { policy, isValid, errors } = evalResult;
-        if (isValid) {
-            matching.push(policy);
-        }
-        else {
-            nonMatching.push(policy);
-            skippedDetails.push({
-                name: policy.name,
-                reason: "Dataset schema not compatible with policy JSON Schema",
-                details: errors,
-            });
-        }
-    }
-    return { matching, nonMatching, skippedDetails };
-}
-/**
- * Helper that evaluates JSON-Schema-based policies and returns branch-level
- * matches plus scheduling information.
- *
- * This is intended to be consumed by a separate SQL builder layer or for
- * inspection of branch-level match details.
- *
- * @param dataset - The dataset against which policies should be evaluated.
- * @param policies - JSON-Schema-based connector policies.
- * @param attributes - Attributes associated with the dataset.
- * @returns JsonSchemaPolicyMatchResult with applied/skipped policies and branch matches.
- * @throws If no policies are compatible with the dataset.
- */
-export function buildPolicySqlWithJsonSchema(dataset, policies, attributes) {
-    const evaluations = evaluateJsonSchemaPolicies(dataset, attributes, policies);
-    const appliedPolicies = [];
-    const skippedPolicies = [];
-    const warnings = [];
-    const branchMatches = [];
-    for (const evalResult of evaluations) {
-        const { policy, isValid, matches, errors } = evalResult;
-        if (!isValid) {
-            skippedPolicies.push({
-                name: policy.name,
-                reason: "Dataset schema not compatible with policy JSON Schema",
-                details: errors,
-            });
-            continue;
-        }
-        appliedPolicies.push(policy.name);
-        branchMatches.push({
-            policyName: policy.name,
-            branches: matches,
-        });
-    }
-    if (appliedPolicies.length === 0) {
-        const errorMessage = policies.length === 0
-            ? "No policies provided"
-            : `No policies are compatible with dataset "${dataset.display_name || dataset.name}".`;
-        const details = skippedPolicies.length > 0
-            ? [`Skipped policies: ${skippedPolicies.map((p) => p.name).join(", ")}`]
-            : [];
-        throw new Error(`${errorMessage}\n${details.join("\n")}`);
-    }
-    if (skippedPolicies.length > 0) {
-        warnings.push(`Skipped ${skippedPolicies.length} incompatible policies: ${skippedPolicies
-            .map((p) => p.name)
-            .join(", ")}`);
-    }
-    const refreshSchedules = evaluations
-        .filter((e) => e.isValid && e.policy.metadata.refresh_schedule?.max)
-        .map((e) => e.policy.metadata.refresh_schedule.max);
-    const refresh_schedule = getSmallestRefreshSchedule(refreshSchedules);
-    return {
-        appliedPolicies,
-        skippedPolicies,
-        warnings,
-        refresh_schedule,
-        branchMatches,
-    };
-}
-/**
- * Uses JSON Schema branch matching to determine valid policies, then builds
- * SELECT and WHERE SQL fragments.
- *
- * @param dataset - Dataset for which to evaluate policies and produce SQL.
- * @param policies - JSON-Schema-based connector policies.
- * @param attributes - Attributes associated with the dataset.
- * @returns PolicyMatchResult with select, where, applied/skipped policies, warnings, and refresh schedule.
- */
-export function buildPolicySqlWithValidation(dataset, policies, attributes) {
-    const matchResult = buildPolicySqlWithJsonSchema(dataset, policies, attributes);
-    const attributeByName = new Map();
-    for (const attr of attributes) {
-        attributeByName.set(attr.name, attr);
-    }
-    const sqlFragments = buildSqlFromBranchMatches(matchResult.branchMatches, attributeByName, dataset.name);
-    return {
-        ...sqlFragments,
-        appliedPolicies: matchResult.appliedPolicies,
-        skippedPolicies: matchResult.skippedPolicies,
-        warnings: matchResult.warnings,
-        refresh_schedule: matchResult.refresh_schedule,
-    };
-}
-/**
- * Build SQL fragments without validation. Assumes you already filtered the policies
- * however you want (e.g. via categorizePolicies). Builds SELECT and WHERE SQL
- * fragments from all provided policies.
- *
- * @param dataset - Dataset for which to build SQL fragments.
- * @param policies - JSON-Schema-based connector policies.
- * @param attributes - Attributes associated with the dataset.
- * @returns PolicySqlFragments with select and where arrays.
- */
-export function buildPolicySql(dataset, policies, attributes) {
-    const evaluations = evaluateJsonSchemaPolicies(dataset, attributes, policies);
-    const attributeByName = new Map();
-    for (const attr of attributes) {
-        attributeByName.set(attr.name, attr);
-    }
-    const branchMatches = [];
-    for (const evalResult of evaluations) {
-        if (evalResult.isValid) {
-            branchMatches.push({
-                policyName: evalResult.policy.name,
-                branches: evalResult.matches,
-            });
-        }
-    }
-    return buildSqlFromBranchMatches(branchMatches, attributeByName, dataset.name);
-}