db4ai 0.3.0 → 0.3.1

package/dist/cli/seed/index.js

@@ -1,774 +1,13 @@
-/**
- * Module 15: Seed Runner
- *
- * Executes static and generated seeds with progress tracking.
- *
- * Two seed types:
- * - Static (Seed()): Load from API or inline data, apply transforms, mark readOnly
- * - Generated (Generate()): Use LLM, apply constraints/variations, handle cascading
- */
-export class SeedError extends Error {
-    seedName;
-    type;
-    position;
-    constructor(message, context) {
-        super(message);
-        this.name = 'SeedError';
-        if (context) {
-            this.seedName = context.seedName;
-            this.type = context.type;
-            this.position = context.position;
-        }
-    }
-}
-// =============================================================================
-// Helper Functions
-// =============================================================================
-export function Seed(input) {
-    if (!input.data && !input.source) {
-        throw new SeedError('Static seed must have data or source');
-    }
-    return {
-        kind: 'static',
-        type: input.type,
-        data: input.data,
-        source: input.source,
-        headers: input.headers,
-        dataPath: input.dataPath,
-        nextPath: input.nextPath,
-        retries: input.retries,
-        transform: input.transform,
-        fieldMapping: input.fieldMapping,
-        readOnly: input.readOnly,
-        concurrency: input.concurrency,
-    };
-}
-export function Generate(input) {
-    // Default count to 1 if neither count nor ids is provided
-    const count = input.count ?? (input.ids ? undefined : 1);
-    // Validate that at least one of count or ids is resolvable
-    // Only throw if after defaulting, we still don't have a valid count
-    if (count === undefined && !input.ids) {
-        throw new SeedError('Generated seed must have count or ids');
-    }
-    return {
-        kind: 'generated',
-        type: input.type,
-        count,
-        ids: input.ids,
-        prompt: input.prompt,
-        constraints: input.constraints,
-        variations: input.variations,
-        cascade: input.cascade,
-        cascadeDepth: input.cascadeDepth,
-        cascadeReuse: input.cascadeReuse,
-        readOnly: input.readOnly,
-        concurrency: input.concurrency,
-    };
-}
-// =============================================================================
-// Implementation
-// =============================================================================
-export function createSeedRunner(db, options) {
-    const { seeds, onProgress, onCheckpoint, onWarning, onEntity, beforeSeed, afterSeed, continueOnError = false, timeout, order, validateSchema = false, concurrency: globalConcurrency = 1, dryRun = false, skipExisting = false, storage, persist = false, } = options;
-    // Helper: Extract references from schema type definition
-    function extractReferences(typeDef) {
-        const refs = [];
-        if (!typeDef || typeof typeDef !== 'object')
-            return refs;
-        for (const [field, value] of Object.entries(typeDef)) {
-            if (typeof value === 'string') {
-                // Match patterns like '->Type', '~>Type', '->Type[]', '->Type?'
-                const hardMatch = value.match(/^->([A-Z][a-zA-Z0-9]*)/);
-                if (hardMatch) {
-                    refs.push({ type: hardMatch[1], fuzzy: false, field });
-                }
-                const fuzzyMatch = value.match(/^~>([A-Z][a-zA-Z0-9]*)/);
-                if (fuzzyMatch) {
-                    refs.push({ type: fuzzyMatch[1], fuzzy: true, field });
-                }
-            }
-        }
-        return refs;
-    }
-    // Helper: Get reference type names only (for backward compatibility)
-    function extractReferenceTypes(typeDef) {
-        return extractReferences(typeDef).map(r => r.type);
-    }
-    // Helper: Check if type is readOnly
-    function isReadOnlyType(typeName) {
-        const typeDef = db.schema[typeName];
-        if (typeDef && typeof typeDef === 'object' && '_readOnly' in typeDef) {
-            return !!typeDef._readOnly;
-        }
-        return false;
-    }
-    // Helper: Get seed's type name
-    function getSeedType(config) {
-        return config.type;
-    }
-    // Helper: Topological sort for dependency order
-    function computeExecutionOrder() {
-        const seedNames = Object.keys(seeds);
-        // Build dependency graph
-        const deps = new Map();
-        const typeToSeed = new Map();
-        for (const name of seedNames) {
-            deps.set(name, new Set());
-            typeToSeed.set(getSeedType(seeds[name]), name);
-        }
-        // Add dependencies based on schema references
-        for (const name of seedNames) {
-            const config = seeds[name];
-            const typeDef = db.schema[config.type];
-            const refs = extractReferenceTypes(typeDef);
-            for (const ref of refs) {
-                const depSeed = typeToSeed.get(ref);
-                if (depSeed && depSeed !== name) {
-                    deps.get(name).add(depSeed);
-                }
-            }
-        }
-        // Static seeds should come first (add implicit dependencies)
-        const staticSeeds = seedNames.filter((n) => seeds[n].kind === 'static');
-        const generatedSeeds = seedNames.filter((n) => seeds[n].kind === 'generated');
-        for (const gen of generatedSeeds) {
-            for (const stat of staticSeeds) {
-                if (!deps.get(gen).has(stat)) {
-                    // Only add if there's a potential reference or just ordering preference
-                    deps.get(gen).add(stat);
-                }
-            }
-        }
-        // Topological sort with cycle detection
-        const result = [];
-        const visited = new Set();
-        const visiting = new Set();
-        function visit(name) {
-            if (visited.has(name))
-                return;
-            if (visiting.has(name)) {
-                throw new SeedError(`Circular dependency detected involving seed "${name}"`);
-            }
-            visiting.add(name);
-            for (const dep of deps.get(name) || []) {
-                visit(dep);
-            }
-            visiting.delete(name);
-            visited.add(name);
-            result.push(name);
-        }
-        for (const name of seedNames) {
-            visit(name);
-        }
-        return result;
-    }
-    // Helper: Apply field mapping
-    function applyFieldMapping(data, mapping) {
-        const result = {};
-        for (const [from, to] of Object.entries(mapping)) {
-            if (from in data) {
-                result[to] = data[from];
-            }
-        }
-        // Copy unmapped fields
-        for (const [key, value] of Object.entries(data)) {
-            if (!(key in mapping)) {
-                result[key] = value;
-            }
-        }
-        return result;
-    }
-    // Helper: Extract data from JSON using simple path
-    function extractDataPath(data, path) {
-        if (!path.startsWith('$.'))
-            return Array.isArray(data) ? data : [];
-        const key = path.slice(2);
-        if (data && typeof data === 'object' && key in data) {
-            const result = data[key];
-            return Array.isArray(result) ? result : [];
-        }
-        return [];
-    }
-    // Helper: Extract next URL from response
-    function extractNextPath(data, path) {
-        if (!path.startsWith('$.'))
-            return null;
-        const key = path.slice(2);
-        if (data && typeof data === 'object' && key in data) {
-            const result = data[key];
-            return typeof result === 'string' ? result : null;
-        }
-        return null;
-    }
-    // Helper: Fetch with retry
-    async function fetchWithRetry(url, headers, retries = 1) {
-        let lastError;
-        for (let i = 0; i < retries; i++) {
-            try {
-                // Only pass options if we have headers to avoid passing undefined
-                const response = headers
-                    ? await fetch(url, { headers })
-                    : await fetch(url);
-                if (response.ok)
-                    return response;
-                if (i === retries - 1) {
-                    throw new SeedError(`API request failed: ${response.status} ${response.statusText}`);
-                }
-            }
-            catch (error) {
-                lastError = error;
-                if (i === retries - 1) {
-                    throw lastError;
-                }
-            }
-        }
-        throw lastError || new SeedError('Fetch failed');
-    }
-    // Helper: Run with timeout
-    function withTimeout(promise, ms) {
-        if (!ms)
-            return promise;
-        return new Promise((resolve, reject) => {
-            const timer = setTimeout(() => {
-                reject(new SeedError('Operation timeout exceeded'));
-            }, ms);
-            promise
-                .then((result) => {
-                clearTimeout(timer);
-                resolve(result);
-            })
-                .catch((error) => {
-                clearTimeout(timer);
-                reject(error);
-            });
-        });
-    }
-    // Helper: Run tasks with concurrency limit
-    async function runWithConcurrency(items, fn, limit) {
-        const results = new Array(items.length);
-        let currentIndex = 0;
-        async function runNext() {
-            while (currentIndex < items.length) {
-                const index = currentIndex++;
-                results[index] = await fn(items[index], index);
-            }
-        }
-        const workers = Array.from({ length: Math.min(limit, items.length) }, () => runNext());
-        await Promise.all(workers);
-        return results;
-    }
-    // Helper: Select variation based on weights
-    function selectVariation(variations, index, total) {
-        if (variations.length === 0)
-            return {};
-        // Normalize weights
-        const totalWeight = variations.reduce((sum, v) => sum + (v.weight ?? 1), 0);
-        const normalizedWeights = variations.map((v) => (v.weight ?? 1) / totalWeight);
-        // Distribute evenly based on index when possible
-        let cumulative = 0;
-        const position = index / total;
-        for (let i = 0; i < variations.length; i++) {
-            cumulative += normalizedWeights[i];
-            if (position < cumulative || i === variations.length - 1) {
-                const { weight: _, ...rest } = variations[i];
-                return rest;
-            }
-        }
-        const { weight: _, ...rest } = variations[0];
-        return rest;
-    }
-    // Helper: Cascade generate referenced entities recursively
-    async function cascadeGenerate(typeName, result, remainingDepth, reuse, visited) {
-        if (remainingDepth <= 0)
-            return;
-        const typeDef = db.schema[typeName];
-        const refs = extractReferences(typeDef);
-        for (const ref of refs) {
-            // Skip already visited types to prevent infinite loops
-            const visitKey = `${typeName}:${ref.field}:${ref.type}`;
-            if (visited.has(visitKey))
-                continue;
-            visited.add(visitKey);
-            // For fuzzy references, list existing entities for matching
-            // This happens even for readOnly types since we're matching, not generating
-            if (ref.fuzzy) {
-                await db.list(ref.type);
-                // Don't generate for fuzzy references, just list for matching
-                continue;
-            }
-            // Skip readOnly types for hard references (we can't generate them)
-            if (isReadOnlyType(ref.type))
-                continue;
-            // Check if we need to create the referenced entity
-            const refId = result && typeof result === 'object'
-                ? result[ref.field]
-                : undefined;
-            if (reuse && refId) {
-                const existing = await db.get(ref.type, refId);
-                if (existing)
-                    continue;
-            }
-            // Generate referenced entity
-            const generatedRef = await db.generate({
-                type: ref.type,
-                prompt: `${ref.type} for ${typeName}`,
-            });
-            // Recursively cascade if we have depth left
-            if (remainingDepth > 1) {
-                await cascadeGenerate(ref.type, generatedRef, remainingDepth - 1, reuse, visited);
-            }
-        }
-    }
-    // Helper: Validate schema
-    function validateSchemaField(typeName, fieldName, strict) {
-        const typeDef = db.schema[typeName];
-        if (!typeDef || typeof typeDef !== 'object') {
-            if (strict) {
-                throw new SeedError(`Type "${typeName}" not found in schema`);
-            }
-            return false;
-        }
-        if (fieldName === 'weight')
-            return true; // Special case for variations
-        if (!(fieldName in typeDef)) {
-            if (strict) {
-                throw new SeedError(`Field "${fieldName}" not found in ${typeName}`);
-            }
-            if (onWarning) {
-                onWarning(`Field "${fieldName}" not found in ${typeName}`);
-            }
-            return false;
-        }
-        return true;
-    }
-    // Run a single seed
-    async function runSeed(seedName, config, runOpts) {
-        const startTime = Date.now();
-        const errors = [];
-        let created = 0;
-        let skipped = 0;
-        let partial = false;
-        let resumeToken;
-        // Parse resume token
-        let startPosition = 0;
-        if (runOpts?.resume) {
-            try {
-                const resumeData = JSON.parse(Buffer.from(runOpts.resume, 'base64').toString());
-                if (resumeData[seedName]?.position) {
-                    startPosition = resumeData[seedName].position;
-                }
-            }
-            catch {
-                // Invalid resume token, start from beginning
-            }
-        }
-        // Validate type exists in schema
-        if (validateSchema === true) {
-            if (!db.schema[config.type]) {
-                throw new SeedError(`Type "${config.type}" not found in schema`, {
-                    seedName,
-                    type: config.type,
-                });
-            }
-        }
-        const phase = config.kind === 'static' ? 'static' : 'generated';
-        if (config.kind === 'static') {
-            // Static seed
-            let data;
-            if (config.data) {
-                data = [...config.data];
-                // Apply transform to inline data
-                if (config.transform) {
-                    data = (await config.transform(data));
-                }
-            }
-            else if (config.source) {
-                // Fetch from API
-                let allData = [];
-                let url = config.source;
-                while (url) {
-                    const response = await fetchWithRetry(url, config.headers, config.retries || 1);
-                    const json = await response.json();
-                    // If there's a transform, apply it to the raw response
-                    // The transform is responsible for extracting the data array
-                    if (config.transform) {
-                        const transformed = (await config.transform(json));
-                        allData = allData.concat(transformed);
-                        // Check for pagination after transform
-                        if (config.nextPath) {
-                            url = extractNextPath(json, config.nextPath);
-                        }
-                        else {
-                            url = null;
-                        }
-                    }
-                    else if (config.dataPath) {
-                        const pageData = extractDataPath(json, config.dataPath);
-                        allData = allData.concat(pageData);
-                        // Check for pagination
-                        if (config.nextPath) {
-                            url = extractNextPath(json, config.nextPath);
-                        }
-                        else {
-                            url = null;
-                        }
-                    }
-                    else if (Array.isArray(json)) {
-                        allData = allData.concat(json);
-                        url = null;
-                    }
-                    else if (json.data && Array.isArray(json.data)) {
-                        allData = allData.concat(json.data);
-                        url = null;
-                    }
-                    else {
-                        allData.push(json);
-                        url = null;
-                    }
-                }
-                data = allData;
-            }
-            else {
-                data = [];
-            }
-            // Apply field mapping
-            if (config.fieldMapping) {
-                data = data.map((item) => applyFieldMapping(item, config.fieldMapping));
-            }
-            const total = data.length;
-            const concurrency = config.concurrency ?? globalConcurrency;
-            // Process items
-            const processItem = async (item, index) => {
-                const position = index + 1;
-                // Check if entity already exists
-                const id = item.id;
-                if (id) {
-                    const existing = await db.get(config.type, id);
-                    if (existing) {
-                        skipped++;
-                        return;
-                    }
-                }
-                // Add readOnly flag if needed
-                const entityData = config.readOnly ? { ...item, _readOnly: true } : { ...item };
-                // Generate ID if not provided
-                if (!entityData.id) {
-                    const name = entityData.name;
-                    entityData.id = name
-                        ? name.toLowerCase().replace(/\s+/g, '-')
-                        : `${config.type.toLowerCase()}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
-                }
-                if (!dryRun) {
-                    await db.create(config.type, entityData);
-                    if (onEntity) {
-                        onEntity(config.type, entityData);
-                    }
-                }
-                created++;
-                if (onCheckpoint) {
-                    onCheckpoint(position);
-                }
-                if (onProgress) {
-                    onProgress({
-                        seedName,
-                        type: config.type,
-                        phase,
-                        current: position,
-                        total,
-                        percentage: Math.round((position / total) * 100),
-                        rate: position / ((Date.now() - startTime) / 1000),
-                        eta: total > position
-                            ? Math.round(((Date.now() - startTime) / position) * (total - position))
-                            : 0,
-                    });
-                }
-                if (persist && storage) {
-                    await storage.set(`seed:${seedName}:state`, JSON.stringify({ position, total }));
-                }
-            };
-            if (concurrency > 1) {
-                await runWithConcurrency(data, processItem, concurrency);
-            }
-            else {
-                for (let i = 0; i < data.length; i++) {
-                    await processItem(data[i], i);
-                }
-            }
-        }
-        else {
-            // Generated seed
-            const count = config.ids?.length ?? config.count ?? 1;
-            const total = count - startPosition;
-            const concurrency = config.concurrency ?? globalConcurrency;
-            // Validate constraints
-            if (validateSchema && config.constraints) {
-                const strict = validateSchema === true;
-                for (const field of Object.keys(config.constraints)) {
-                    validateSchemaField(config.type, field, strict);
-                }
-            }
-            // Validate variations
-            if (validateSchema && config.variations) {
-                const strict = validateSchema === true;
-                for (const variation of config.variations) {
-                    for (const field of Object.keys(variation)) {
-                        if (field !== 'weight') {
-                            validateSchemaField(config.type, field, strict);
-                        }
-                    }
-                }
-            }
-            // Resolve reference constraints
-            let resolvedConstraints = config.constraints ? { ...config.constraints } : undefined;
-            if (resolvedConstraints) {
-                for (const [key, value] of Object.entries(resolvedConstraints)) {
-                    if (value && typeof value === 'object' && 'ref' in value) {
-                        const refType = value.ref;
-                        await db.list(refType);
-                        // Keep constraint as-is for generation
-                    }
-                }
-            }
-            // Generate entities
-            const generateEntity = async (index) => {
-                const actualIndex = startPosition + index;
-                const position = actualIndex + 1;
-                // Get ID (from ids array or generate)
-                const id = config.ids?.[actualIndex];
-                // Check if entity already exists
-                if (skipExisting && id) {
-                    const existing = await db.get(config.type, id);
-                    if (existing) {
-                        skipped++;
-                        return;
-                    }
-                }
-                // Apply variation
-                let variationProps = {};
-                if (config.variations && config.variations.length > 0) {
-                    variationProps = selectVariation(config.variations, actualIndex, count);
-                }
-                try {
-                    const generateConfig = {
-                        type: config.type,
-                        prompt: config.prompt,
-                        constraints: resolvedConstraints,
-                        ...variationProps,
-                    };
-                    if (id) {
-                        generateConfig.id = id;
-                    }
-                    if (!dryRun) {
-                        const result = await withTimeout(db.generate(generateConfig), timeout);
-                        // Handle cascading
-                        if (config.cascade) {
-                            await cascadeGenerate(config.type, result, config.cascadeDepth ?? Infinity, config.cascadeReuse ?? false, new Set());
-                        }
-                        if (onEntity) {
-                            onEntity(config.type, result);
-                        }
-                    }
-                    created++;
-                    if (onCheckpoint) {
-                        onCheckpoint(position);
-                    }
-                    if (onProgress) {
-                        onProgress({
-                            seedName,
-                            type: config.type,
-                            phase,
-                            current: position,
-                            total: count,
-                            percentage: Math.round((position / count) * 100),
-                            rate: position / ((Date.now() - startTime) / 1000),
-                            eta: count > position
-                                ? Math.round(((Date.now() - startTime) / position) * (count - position))
-                                : 0,
-                        });
-                    }
-                    if (persist && storage) {
-                        await storage.set(`seed:${seedName}:state`, JSON.stringify({ position, total: count }));
-                    }
-                }
-                catch (error) {
-                    const errorMessage = error instanceof Error ? error.message : String(error);
-                    const errorInfo = {
-                        message: errorMessage,
-                        position,
-                    };
-                    // Check for interrupt
-                    if (error &&
-                        typeof error === 'object' &&
-                        'code' in error &&
-                        error.code === 'INTERRUPTED') {
-                        partial = true;
-                        resumeToken = Buffer.from(JSON.stringify({ [seedName]: { position } })).toString('base64');
-                        errors.push(errorInfo);
-                        return;
-                    }
-                    if (continueOnError) {
-                        errors.push(errorInfo);
-                    }
-                    else {
-                        throw new SeedError(errorMessage, {
-                            seedName,
-                            type: config.type,
-                            position,
-                        });
-                    }
-                }
-            };
-            if (concurrency > 1) {
-                const indices = Array.from({ length: total }, (_, i) => i);
-                await runWithConcurrency(indices, generateEntity, concurrency);
-            }
-            else {
-                for (let i = 0; i < total; i++) {
-                    await generateEntity(i);
-                }
-            }
-        }
-        return {
-            success: errors.length === 0,
-            created,
-            skipped,
-            errors,
-            partial,
-            duration: Date.now() - startTime,
-            resumeToken,
-        };
-    }
-    // Run single seed
-    async function run(seedName, runOpts) {
-        const config = seeds[seedName];
-        if (!config) {
-            throw new SeedError(`Seed "${seedName}" not found`, { seedName });
-        }
-        // Validate config
-        if (!config.kind) {
-            throw new SeedError(`Invalid seed configuration for "${seedName}"`, { seedName });
-        }
-        // Call beforeSeed hook
-        let effectiveConfig = config;
-        if (beforeSeed) {
-            const hookResult = beforeSeed(seedName, config);
-            if (hookResult === null) {
-                // Skip this seed
-                return {
-                    success: true,
-                    created: 0,
-                    skipped: 0,
-                    errors: [],
-                    partial: false,
-                    duration: 0,
-                };
-            }
-            if (hookResult) {
-                effectiveConfig = hookResult;
-            }
-        }
-        const result = await runSeed(seedName, effectiveConfig, runOpts);
-        // Call afterSeed hook
-        if (afterSeed) {
-            afterSeed(seedName, result);
-        }
-        return result;
-    }
-    // Run all seeds
-    async function runAll() {
-        const startTime = Date.now();
-        const executionOrder = order ?? computeExecutionOrder();
-        if (dryRun) {
-            const wouldCreate = {};
-            for (const seedName of executionOrder) {
-                const config = seeds[seedName];
-                // Validate type exists in schema
-                if (validateSchema === true) {
-                    if (!db.schema[config.type]) {
-                        throw new SeedError(`Type "${config.type}" not found in schema`, {
-                            seedName,
-                            type: config.type,
-                        });
-                    }
-                }
-                if (config.kind === 'static') {
-                    wouldCreate[config.type] = config.data?.length ?? 0;
-                }
-                else {
-                    wouldCreate[config.type] = config.ids?.length ?? config.count ?? 1;
-                }
-            }
-            return {
-                success: true,
-                created: 0,
-                skipped: 0,
-                errors: [],
-                partial: false,
-                duration: Date.now() - startTime,
-                dryRun: true,
-                wouldCreate,
-                executionOrder,
-            };
-        }
-        let totalCreated = 0;
-        let totalSkipped = 0;
-        const allErrors = [];
-        for (const seedName of executionOrder) {
-            const result = await run(seedName);
-            totalCreated += result.created;
-            totalSkipped += result.skipped;
-            allErrors.push(...result.errors);
-        }
-        return {
-            success: allErrors.length === 0,
-            created: totalCreated,
-            skipped: totalSkipped,
-            errors: allErrors,
-            partial: false,
-            duration: Date.now() - startTime,
-        };
-    }
-    // Reset single seed
-    async function reset(seedName, resetOpts) {
-        const config = seeds[seedName];
-        if (!config) {
-            throw new SeedError(`Seed "${seedName}" not found`, { seedName });
-        }
-        // Clear the type
-        await db.clear(config.type);
-        // Clear cascaded types if requested
-        if (resetOpts?.cascade) {
-            const typeDef = db.schema[config.type];
-            const refs = extractReferenceTypes(typeDef);
-            for (const refType of refs) {
-                if (!isReadOnlyType(refType)) {
-                    await db.clear(refType);
-                }
-            }
-        }
-        // Re-run the seed
-        return run(seedName);
-    }
-    // Reset all seeds
-    async function resetAll(resetOpts) {
-        // Clear all types
-        for (const seedName of Object.keys(seeds)) {
-            const config = seeds[seedName];
-            // Skip readOnly if preserveReadOnly is true
-            if (resetOpts?.preserveReadOnly && config.readOnly) {
-                continue;
-            }
-            await db.clear(config.type);
-        }
-        // Run all seeds
-        return runAll();
-    }
-    return {
-        run,
-        runAll,
-        reset,
-        resetAll,
-    };
-}
+import {
+  Generate,
+  Seed,
+  SeedError,
+  createSeedRunner
+} from "../../chunk-EERD6CDF.js";
+export {
+  Generate,
+  Seed,
+  SeedError,
+  createSeedRunner
+};
 //# sourceMappingURL=index.js.map