ctxpkg 0.0.1 → 0.0.4

package/dist/collections/collections.js

@@ -0,0 +1,933 @@
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync, realpathSync, createWriteStream, mkdirSync } from 'node:fs';
+import { readFile, glob, mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { resolve, join, dirname } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { simpleGit } from 'simple-git';
+import * as tar from 'tar';
+import { projectConfigSchema, collectionRecordSchema, manifestSchema, isGlobSources, isFileSources, isGitUrl, parseGitUrl, } from './collections.schemas.js';
+import { DatabaseService, tableNames } from '#root/database/database.js';
+import { DocumentsService } from '#root/documents/documents.js';
+import { config } from '#root/config/config.js';
+class CollectionsService {
+    #services;
+    constructor(services) {
+        this.#services = services;
+    }
+    // === Project Config ===
+    /**
+     * Get the project config file path for a given directory.
+     */
+    getProjectConfigPath = (cwd = process.cwd()) => {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const configFile = config.get('project.configFile');
+        return resolve(cwd, configFile);
+    };
+    /**
+     * Check if a project config file exists.
+     */
+    projectConfigExists = (cwd = process.cwd()) => {
+        return existsSync(this.getProjectConfigPath(cwd));
+    };
+    /**
+     * Read and parse the project config file.
+     */
+    readProjectConfig = (cwd = process.cwd()) => {
+        const configPath = this.getProjectConfigPath(cwd);
+        if (!existsSync(configPath)) {
+            return { collections: {} };
+        }
+        const content = readFileSync(configPath, 'utf-8');
+        const parsed = JSON.parse(content);
+        return projectConfigSchema.parse(parsed);
+    };
+    /**
+     * Write the project config file.
+     */
+    writeProjectConfig = (projectConfig, cwd = process.cwd()) => {
+        const configPath = this.getProjectConfigPath(cwd);
+        const content = JSON.stringify(projectConfig, null, 2);
+        writeFileSync(configPath, content, 'utf-8');
+    };
+    /**
+     * Initialize a new project config file.
+     */
+    initProjectConfig = (cwd = process.cwd(), force = false) => {
+        const configPath = this.getProjectConfigPath(cwd);
+        if (existsSync(configPath) && !force) {
+            throw new Error(`Project config already exists at ${configPath}`);
+        }
+        const initialConfig = { collections: {} };
+        this.writeProjectConfig(initialConfig, cwd);
+    };
+    // === Global Config ===
+    /**
+     * Get the global config file path.
+     */
+    getGlobalConfigPath = () => {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        return config.get('global.configFile');
+    };
+    /**
+     * Check if the global config file exists.
+     */
+    globalConfigExists = () => {
+        return existsSync(this.getGlobalConfigPath());
+    };
+    /**
+     * Read and parse the global config file.
+     */
+    readGlobalConfig = () => {
+        const configPath = this.getGlobalConfigPath();
+        if (!existsSync(configPath)) {
+            return { collections: {} };
+        }
+        const content = readFileSync(configPath, 'utf-8');
+        const parsed = JSON.parse(content);
+        return projectConfigSchema.parse(parsed);
+    };
+    /**
+     * Write the global config file. Auto-creates directory if needed.
+     */
+    writeGlobalConfig = (globalConfig) => {
+        const configPath = this.getGlobalConfigPath();
+        const configDir = dirname(configPath);
+        // Ensure directory exists
+        if (!existsSync(configDir)) {
+            mkdirSync(configDir, { recursive: true });
+        }
+        const content = JSON.stringify(globalConfig, null, 2);
+        writeFileSync(configPath, content, 'utf-8');
+    };
+    // === Unified Config Operations ===
+    /**
+     * Add a collection to project or global config.
+     */
+    addToConfig = (name, spec, options = {}) => {
+        const { global: isGlobal = false, cwd = process.cwd() } = options;
+        if (isGlobal) {
+            const globalConfig = this.readGlobalConfig();
+            if (name in globalConfig.collections) {
+                throw new Error(`Collection "${name}" already exists in global config`);
+            }
+            globalConfig.collections[name] = spec;
+            this.writeGlobalConfig(globalConfig);
+        }
+        else {
+            this.addToProjectConfig(name, spec, cwd);
+        }
+    };
+    /**
+     * Remove a collection from project or global config.
+     */
+    removeFromConfig = (name, options = {}) => {
+        const { global: isGlobal = false, cwd = process.cwd() } = options;
+        if (isGlobal) {
+            const globalConfig = this.readGlobalConfig();
+            if (!(name in globalConfig.collections)) {
+                throw new Error(`Collection "${name}" not found in global config`);
+            }
+            const { [name]: _removed, ...rest } = globalConfig.collections;
+            void _removed;
+            globalConfig.collections = rest;
+            this.writeGlobalConfig(globalConfig);
+        }
+        else {
+            this.removeFromProjectConfig(name, cwd);
+        }
+    };
+    /**
+     * Get a collection spec by name from project or global config.
+     * If global is not specified, searches local first then global.
+     */
+    getFromConfig = (name, options = {}) => {
+        const { global: isGlobal, cwd = process.cwd() } = options;
+        if (isGlobal === true) {
+            const globalConfig = this.readGlobalConfig();
+            return globalConfig.collections[name] || null;
+        }
+        if (isGlobal === false) {
+            return this.getFromProjectConfig(name, cwd);
+        }
+        // If global is undefined, search local first then global
+        const localSpec = this.getFromProjectConfig(name, cwd);
+        if (localSpec) {
+            return localSpec;
+        }
+        const globalConfig = this.readGlobalConfig();
+        return globalConfig.collections[name] || null;
+    };
+    /**
+     * Get all collections from both local and global configs.
+     * Returns a map with collection name as key and spec + source info as value.
+     * Local collections take precedence over global ones with the same name.
+     */
+    getAllCollections = (cwd = process.cwd()) => {
+        const result = new Map();
+        // Add global collections first
+        const globalConfig = this.readGlobalConfig();
+        for (const [name, spec] of Object.entries(globalConfig.collections)) {
+            result.set(name, { spec, source: 'global' });
+        }
+        // Add local collections (will override global ones with same name)
+        if (this.projectConfigExists(cwd)) {
+            const projectConfig = this.readProjectConfig(cwd);
+            for (const [name, spec] of Object.entries(projectConfig.collections)) {
+                result.set(name, { spec, source: 'local' });
+            }
+        }
+        return result;
+    };
+    // === Collection ID Computation ===
+    /**
+     * Normalize a path to its canonical absolute form.
+     */
+    normalizePath = (path, basePath = process.cwd()) => {
+        const absolutePath = resolve(basePath, path);
+        // Resolve symlinks to canonical path
+        try {
+            return realpathSync(absolutePath);
+        }
+        catch {
+            // Path doesn't exist yet, return resolved path
+            return absolutePath;
+        }
+    };
+    /**
+     * Compute the collection ID for a given spec.
+     * Format: pkg:{normalized_url}
+     */
+    computeCollectionId = (spec) => {
+        // Normalize URL (remove trailing slashes)
+        const normalizedUrl = spec.url.replace(/\/+$/, '');
+        return `pkg:${normalizedUrl}`;
+    };
+    // === Database Operations ===
+    /**
+     * Get a collection record by ID.
+     */
+    getCollection = async (id) => {
+        const databaseService = this.#services.get(DatabaseService);
+        const database = await databaseService.getInstance();
+        const [record] = await database(tableNames.collections).where({ id }).limit(1);
+        if (!record) {
+            return null;
+        }
+        return collectionRecordSchema.parse(record);
+    };
+    /**
+     * List all collection records.
+     */
+    listCollections = async () => {
+        const databaseService = this.#services.get(DatabaseService);
+        const database = await databaseService.getInstance();
+        const records = await database(tableNames.collections).orderBy('created_at', 'asc');
+        return records.map((record) => collectionRecordSchema.parse(record));
+    };
+    /**
+     * Create or update a collection record.
+     */
+    upsertCollection = async (id, data) => {
+        const databaseService = this.#services.get(DatabaseService);
+        const database = await databaseService.getInstance();
+        const now = new Date().toISOString();
+        const existing = await this.getCollection(id);
+        if (existing) {
+            await database(tableNames.collections)
+                .where({ id })
+                .update({
+                ...data,
+                updated_at: now,
+            });
+        }
+        else {
+            await database(tableNames.collections).insert({
+                id,
+                ...data,
+                created_at: now,
+                updated_at: now,
+            });
+        }
+    };
+    /**
+     * Delete a collection record.
+     */
+    deleteCollection = async (id) => {
+        const databaseService = this.#services.get(DatabaseService);
+        const database = await databaseService.getInstance();
+        await database(tableNames.collections).where({ id }).delete();
+    };
+    /**
+     * Update the last sync timestamp for a collection.
+     */
+    updateLastSync = async (id) => {
+        const databaseService = this.#services.get(DatabaseService);
+        const database = await databaseService.getInstance();
+        const now = new Date().toISOString();
+        await database(tableNames.collections).where({ id }).update({
+            last_sync_at: now,
+            updated_at: now,
+        });
+    };
+    /**
+     * Update the manifest hash for a collection.
+     */
+    updateManifestHash = async (id, hash) => {
+        const databaseService = this.#services.get(DatabaseService);
+        const database = await databaseService.getInstance();
+        const now = new Date().toISOString();
+        await database(tableNames.collections).where({ id }).update({
+            manifest_hash: hash,
+            updated_at: now,
+        });
+    };
+    // === Project Config Helpers ===
+    /**
+     * Add a collection to the project config.
+     */
+    addToProjectConfig = (name, spec, cwd = process.cwd()) => {
+        const projectConfig = this.readProjectConfig(cwd);
+        if (name in projectConfig.collections) {
+            throw new Error(`Collection "${name}" already exists in project config`);
+        }
+        projectConfig.collections[name] = spec;
+        this.writeProjectConfig(projectConfig, cwd);
+    };
+    /**
+     * Remove a collection from the project config.
+     */
+    removeFromProjectConfig = (name, cwd = process.cwd()) => {
+        const projectConfig = this.readProjectConfig(cwd);
+        if (!(name in projectConfig.collections)) {
+            throw new Error(`Collection "${name}" not found in project config`);
+        }
+        const { [name]: _removed, ...rest } = projectConfig.collections;
+        void _removed; // Intentionally unused
+        projectConfig.collections = rest;
+        this.writeProjectConfig(projectConfig, cwd);
+    };
+    /**
+     * Get a collection spec by name from the project config.
+     */
+    getFromProjectConfig = (name, cwd = process.cwd()) => {
+        const projectConfig = this.readProjectConfig(cwd);
+        return projectConfig.collections[name] || null;
+    };
+    // === Sync Status ===
+    /**
+     * Get sync status for a collection by computing its ID and checking the database.
+     */
+    getSyncStatus = async (spec) => {
+        const id = this.computeCollectionId(spec);
+        const record = await this.getCollection(id);
+        if (!record || !record.last_sync_at) {
+            return 'not_synced';
+        }
+        // For now, just check if it has ever been synced
+        // Future: compare manifest hashes for staleness
+        return 'synced';
+    };
+    // === Sync Operations ===
+    /**
+     * Sync a collection based on its spec.
+     */
+    syncCollection = async (name, spec, cwd = process.cwd(), options = {}) => {
+        // Check if it's a git URL
+        if (isGitUrl(spec.url)) {
+            return this.syncGitCollection(name, spec, cwd, options);
+        }
+        return this.syncPkgCollection(name, spec, cwd, options);
+    };
+    // === Manifest Handling ===
+    /**
+     * Parse a manifest URL and determine its protocol.
+     */
+    parseManifestUrl = (url, cwd = process.cwd()) => {
+        // Check for git URLs first
+        if (isGitUrl(url)) {
+            const parsed = parseGitUrl(url);
+            return {
+                protocol: 'git',
+                cloneUrl: parsed.cloneUrl,
+                ref: parsed.ref,
+                manifestPath: parsed.manifestPath,
+                isBundle: false,
+            };
+        }
+        const isBundle = url.endsWith('.tar.gz') || url.endsWith('.tgz');
+        if (url.startsWith('file://')) {
+            const filePath = url.slice(7); // Remove 'file://'
+            const resolvedPath = this.normalizePath(filePath, cwd);
+            return { protocol: 'file', path: resolvedPath, isBundle };
+        }
+        if (url.startsWith('https://') || url.startsWith('http://')) {
+            return { protocol: 'https', path: url, isBundle };
+        }
+        // Assume it's a relative file path
+        const resolvedPath = this.normalizePath(url, cwd);
+        return { protocol: 'file', path: resolvedPath, isBundle };
+    };
+    /**
+     * Load a manifest from a file:// URL.
+     */
+    loadLocalManifest = async (manifestPath) => {
+        const content = await readFile(manifestPath, 'utf8');
+        const parsed = JSON.parse(content);
+        return manifestSchema.parse(parsed);
+    };
+    /**
+     * Resolve manifest sources to a list of file entries.
+     * For glob sources: expand globs relative to manifest directory.
+     * For files sources: resolve paths relative to manifest or baseUrl.
+     */
+    resolveManifestSources = async (manifest, manifestDir, protocol) => {
+        const sources = manifest.sources;
+        const baseUrl = manifest.baseUrl;
+        if (isGlobSources(sources)) {
+            if (protocol !== 'file') {
+                throw new Error('Glob sources are only supported for file:// manifests');
+            }
+            const entries = [];
+            for (const pattern of sources.glob) {
+                for await (const file of glob(pattern, { cwd: manifestDir })) {
+                    const fullPath = resolve(manifestDir, file);
+                    entries.push({
+                        id: file,
+                        url: `file://${fullPath}`,
+                    });
+                }
+            }
+            return entries;
+        }
+        if (isFileSources(sources)) {
+            return sources.files.map((entry) => this.resolveFileEntry(entry, manifestDir, baseUrl, protocol));
+        }
+        throw new Error('Unknown sources type in manifest');
+    };
+    /**
+     * Resolve a single file entry to its final URL.
+     */
+    resolveFileEntry = (entry, manifestDir, baseUrl, protocol) => {
+        // String shorthand = relative path
+        if (typeof entry === 'string') {
+            return this.resolveFileEntry({ path: entry }, manifestDir, baseUrl, protocol);
+        }
+        // Fully qualified URL
+        if (entry.url) {
+            return {
+                id: entry.url,
+                url: entry.url,
+                hash: entry.hash,
+            };
+        }
+        // Relative path
+        if (entry.path) {
+            let resolvedUrl;
+            if (baseUrl) {
+                // Resolve relative to baseUrl
+                const base = baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
+                resolvedUrl = `${base}${entry.path}`;
+            }
+            else if (protocol === 'file') {
+                // Resolve relative to manifest directory
+                const fullPath = resolve(manifestDir, entry.path);
+                resolvedUrl = `file://${fullPath}`;
+            }
+            else {
+                // For https, resolve relative to manifest URL directory
+                const base = manifestDir.endsWith('/') ? manifestDir : `${manifestDir}/`;
+                resolvedUrl = `${base}${entry.path}`;
+            }
+            return {
+                id: entry.path,
+                url: resolvedUrl,
+                hash: entry.hash,
+            };
+        }
+        throw new Error('File entry must have either path or url');
+    };
+    /**
+     * Fetch content from a URL (file:// or https://).
+     */
+    fetchContent = async (url) => {
+        if (url.startsWith('file://')) {
+            const filePath = url.slice(7);
+            return readFile(filePath, 'utf8');
+        }
+        if (url.startsWith('https://') || url.startsWith('http://')) {
+            const response = await fetch(url);
+            if (!response.ok) {
+                throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+            }
+            return response.text();
+        }
+        throw new Error(`Unsupported URL protocol: ${url}`);
+    };
+    /**
+     * Load a manifest from a remote URL.
+     */
+    loadRemoteManifest = async (manifestUrl) => {
+        const content = await this.fetchContent(manifestUrl);
+        const parsed = JSON.parse(content);
+        const manifest = manifestSchema.parse(parsed);
+        return { manifest, content };
+    };
+    /**
+     * Get the directory part of a URL.
+     */
+    getUrlDirectory = (url) => {
+        const lastSlash = url.lastIndexOf('/');
+        return lastSlash >= 0 ? url.substring(0, lastSlash) : url;
+    };
+    // === Bundle Handling ===
+    /**
+     * Download a bundle to a temporary file and extract it.
+     * Returns the path to the extracted directory.
+     */
+    downloadAndExtractBundle = async (url, onProgress) => {
+        const tempDir = await mkdtemp(join(tmpdir(), 'ai-assist-bundle-'));
+        try {
+            if (url.startsWith('file://')) {
+                // Local bundle - extract directly
+                const bundlePath = url.slice(7);
+                onProgress?.('Extracting local bundle...');
+                await tar.extract({
+                    file: bundlePath,
+                    cwd: tempDir,
+                });
+            }
+            else {
+                // Remote bundle - download then extract
+                onProgress?.('Downloading bundle...');
+                const response = await fetch(url);
+                if (!response.ok) {
+                    throw new Error(`Failed to download bundle: ${response.status} ${response.statusText}`);
+                }
+                const bundlePath = join(tempDir, 'bundle.tar.gz');
+                // Stream response to file
+                if (!response.body) {
+                    throw new Error('Response body is empty');
+                }
+                const fileStream = createWriteStream(bundlePath);
+                await pipeline(Readable.fromWeb(response.body), fileStream);
+                onProgress?.('Extracting bundle...');
+                await tar.extract({
+                    file: bundlePath,
+                    cwd: tempDir,
+                });
+            }
+            // Find the extracted content - could be in root or a subdirectory
+            // Check if manifest.json exists at root or find it
+            const manifestAtRoot = join(tempDir, 'manifest.json');
+            if (existsSync(manifestAtRoot)) {
+                return tempDir;
+            }
+            // Look for manifest in immediate subdirectories (common for tarballs)
+            const { readdir } = await import('node:fs/promises');
+            const entries = await readdir(tempDir, { withFileTypes: true });
+            for (const entry of entries) {
+                if (entry.isDirectory()) {
+                    const subManifest = join(tempDir, entry.name, 'manifest.json');
+                    if (existsSync(subManifest)) {
+                        return join(tempDir, entry.name);
+                    }
+                }
+            }
+            throw new Error('Could not find manifest.json in bundle');
+        }
+        catch (error) {
+            // Clean up on error
+            await rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
+            throw error;
+        }
+    };
+    /**
+     * Sync a bundle collection.
+     */
+    syncBundleCollection = async (name, spec, cwd = process.cwd(), options = {}) => {
+        const { force = false, onProgress } = options;
+        const collectionId = this.computeCollectionId(spec);
+        const parsed = this.parseManifestUrl(spec.url, cwd);
+        // This method only handles file/https bundles, not git URLs
+        if (parsed.protocol === 'git') {
+            throw new Error('syncBundleCollection does not support git URLs');
+        }
+        const { protocol, path: bundlePath } = parsed;
+        // Reconstruct URL with protocol for downloadAndExtractBundle
+        const bundleUrl = protocol === 'file' ? `file://${bundlePath}` : bundlePath;
+        let tempDir = null;
+        try {
+            // Download and extract bundle
+            tempDir = await this.downloadAndExtractBundle(bundleUrl, onProgress);
+            const manifestPath = join(tempDir, 'manifest.json');
+            onProgress?.('Reading manifest...');
+            // Load manifest
+            const manifest = await this.loadLocalManifest(manifestPath);
+            const manifestContent = await readFile(manifestPath, 'utf8');
+            const manifestHash = createHash('sha256').update(manifestContent).digest('hex');
+            // Check if we can skip sync
+            const existingCollection = await this.getCollection(collectionId);
+            if (!force && existingCollection?.manifest_hash === manifestHash) {
+                onProgress?.('Bundle unchanged, skipping sync');
+                return { added: 0, updated: 0, removed: 0, total: 0 };
+            }
+            onProgress?.('Resolving sources...');
+            // Resolve sources (always use 'file' protocol for extracted bundle)
+            const entries = await this.resolveManifestSources(manifest, tempDir, 'file');
+            // Get existing documents from database
+            const documentsService = this.#services.get(DocumentsService);
+            const existingDocs = await documentsService.getDocumentIds(collectionId);
+            const existingMap = new Map(existingDocs.map((doc) => [doc.id, doc.hash]));
+            // Compute changes
+            const toAdd = [];
+            const toUpdate = [];
+            const toRemove = [];
+            for (const entry of entries) {
+                const existingHash = existingMap.get(entry.id);
+                if (!existingHash) {
+                    toAdd.push(entry);
+                }
+                else if (force) {
+                    toUpdate.push(entry);
+                }
+                else if (entry.hash) {
+                    if (existingHash !== entry.hash) {
+                        toUpdate.push(entry);
+                    }
+                }
+                else {
+                    toUpdate.push(entry);
+                }
+            }
+            const currentIds = new Set(entries.map((e) => e.id));
+            for (const [id] of existingMap) {
+                if (!currentIds.has(id)) {
+                    toRemove.push(id);
+                }
+            }
+            // Apply changes
+            if (toRemove.length > 0) {
+                onProgress?.(`Removing ${toRemove.length} deleted documents...`);
+                await documentsService.deleteDocuments(collectionId, toRemove);
+            }
+            const toProcess = [...toAdd, ...toUpdate];
+            let actualUpdated = 0;
+            for (let i = 0; i < toProcess.length; i++) {
+                const entry = toProcess[i];
+                const isNew = toAdd.includes(entry);
+                onProgress?.(`${isNew ? 'Adding' : 'Checking'} ${entry.id} (${i + 1}/${toProcess.length})...`);
+                const content = await this.fetchContent(entry.url);
+                const contentHash = createHash('sha256').update(content).digest('hex');
+                if (!isNew && !force && existingMap.get(entry.id) === contentHash) {
+                    continue;
+                }
+                if (!isNew)
+                    actualUpdated++;
+                await documentsService.updateDocument({
+                    collection: collectionId,
+                    id: entry.id,
+                    content,
+                });
+            }
+            // Update collection record
+            await this.upsertCollection(collectionId, {
+                url: spec.url,
+                name: manifest.name,
+                version: manifest.version,
+                description: manifest.description ?? null,
+                manifest_hash: manifestHash,
+                last_sync_at: new Date().toISOString(),
+            });
+            return {
+                added: toAdd.length,
+                updated: actualUpdated,
+                removed: toRemove.length,
+                total: entries.length,
+            };
+        }
+        finally {
+            // Clean up temp directory
+            if (tempDir) {
+                await rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
+            }
+        }
+    };
+    /**
+     * Sync a git collection.
+     * Clones the repository to a temp directory relative to cwd (to preserve includeIf git config),
+     * reads the manifest from the specified path, and syncs documents.
+     */
+    syncGitCollection = async (name, spec, cwd = process.cwd(), options = {}) => {
+        const { force = false, onProgress } = options;
+        const collectionId = this.computeCollectionId(spec);
+        const parsed = parseGitUrl(spec.url);
+        let tempDir = null;
+        try {
+            // Create temp directory relative to cwd (preserves includeIf git config)
+            const tmpBase = join(cwd, '.ctxpkg', 'tmp');
+            mkdirSync(tmpBase, { recursive: true });
+            // Create unique temp dir
+            const uniqueId = Math.random().toString(36).substring(2, 10);
+            tempDir = join(tmpBase, `git-${uniqueId}`);
+            mkdirSync(tempDir, { recursive: true });
+            // Clone the repository
+            const refDisplay = parsed.ref ? ` @ ${parsed.ref}` : '';
+            onProgress?.(`Cloning ${parsed.cloneUrl}${refDisplay}...`);
+            const git = simpleGit();
+            // Build clone options - disable hooks for security
+            const cloneOptions = ['--config', 'core.hooksPath=/dev/null'];
+            // Use shallow clone when possible (not for commit SHAs)
+            const isCommitSha = parsed.ref && /^[a-f0-9]{7,40}$/i.test(parsed.ref);
+            if (!isCommitSha) {
+                cloneOptions.push('--depth', '1');
+                if (parsed.ref) {
+                    cloneOptions.push('--branch', parsed.ref);
+                }
+            }
+            await git.clone(parsed.cloneUrl, tempDir, cloneOptions);
+            // For commit SHAs, checkout the specific commit
+            if (isCommitSha && parsed.ref) {
+                onProgress?.(`Checking out ${parsed.ref}...`);
+                await simpleGit(tempDir).checkout(parsed.ref);
+            }
+            // Locate manifest
+            const manifestPath = join(tempDir, parsed.manifestPath);
+            if (!existsSync(manifestPath)) {
+                throw new Error(`Manifest not found at ${parsed.manifestPath} in repository`);
+            }
+            onProgress?.('Reading manifest...');
+            // Load manifest
+            const manifest = await this.loadLocalManifest(manifestPath);
+            const manifestContent = await readFile(manifestPath, 'utf8');
+            const manifestHash = createHash('sha256').update(manifestContent).digest('hex');
+            // Check if we can skip sync
+            const existingCollection = await this.getCollection(collectionId);
+            if (!force && existingCollection?.manifest_hash === manifestHash) {
+                onProgress?.('Repository unchanged, skipping sync');
+                return { added: 0, updated: 0, removed: 0, total: 0 };
+            }
+            onProgress?.('Resolving sources...');
+            // Get manifest directory for resolving relative paths
+            const manifestDir = dirname(manifestPath);
+            // Resolve sources (always use 'file' protocol for cloned repo)
+            const entries = await this.resolveManifestSources(manifest, manifestDir, 'file');
+            // Get existing documents from database
+            const documentsService = this.#services.get(DocumentsService);
+            const existingDocs = await documentsService.getDocumentIds(collectionId);
+            const existingMap = new Map(existingDocs.map((doc) => [doc.id, doc.hash]));
+            // Compute changes
+            const toAdd = [];
+            const toUpdate = [];
+            const toRemove = [];
+            for (const entry of entries) {
+                const existingHash = existingMap.get(entry.id);
+                if (!existingHash) {
+                    toAdd.push(entry);
+                }
+                else if (force) {
+                    toUpdate.push(entry);
+                }
+                else if (entry.hash) {
+                    if (existingHash !== entry.hash) {
+                        toUpdate.push(entry);
+                    }
+                }
+                else {
+                    toUpdate.push(entry);
+                }
+            }
+            const currentIds = new Set(entries.map((e) => e.id));
+            for (const [id] of existingMap) {
+                if (!currentIds.has(id)) {
+                    toRemove.push(id);
+                }
+            }
+            // Apply changes
+            if (toRemove.length > 0) {
+                onProgress?.(`Removing ${toRemove.length} deleted documents...`);
+                await documentsService.deleteDocuments(collectionId, toRemove);
+            }
+            const toProcess = [...toAdd, ...toUpdate];
+            let actualUpdated = 0;
+            for (let i = 0; i < toProcess.length; i++) {
+                const entry = toProcess[i];
+                const isNew = toAdd.includes(entry);
+                onProgress?.(`${isNew ? 'Adding' : 'Checking'} ${entry.id} (${i + 1}/${toProcess.length})...`);
+                const content = await this.fetchContent(entry.url);
+                const contentHash = createHash('sha256').update(content).digest('hex');
+                if (!isNew && !force && existingMap.get(entry.id) === contentHash) {
+                    continue;
+                }
+                if (!isNew)
+                    actualUpdated++;
+                await documentsService.updateDocument({
+                    collection: collectionId,
+                    id: entry.id,
+                    content,
+                });
+            }
+            // Update collection record
+            await this.upsertCollection(collectionId, {
+                url: spec.url,
+                name: manifest.name,
+                version: manifest.version,
+                description: manifest.description ?? null,
+                manifest_hash: manifestHash,
+                last_sync_at: new Date().toISOString(),
+            });
+            return {
+                added: toAdd.length,
+                updated: actualUpdated,
+                removed: toRemove.length,
+                total: entries.length,
+            };
+        }
+        finally {
+            // Clean up temp directory
+            if (tempDir) {
+                await rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
+            }
+        }
+    };
+    /**
+     * Sync a pkg collection.
+     */
+    syncPkgCollection = async (name, spec, cwd = process.cwd(), options = {}) => {
+        const { force = false, onProgress } = options;
+        const collectionId = this.computeCollectionId(spec);
+        const parsed = this.parseManifestUrl(spec.url, cwd);
+        // This method only handles file/https, not git URLs
+        if (parsed.protocol === 'git') {
+            throw new Error('syncPkgCollection does not support git URLs');
+        }
+        const { protocol, path: manifestPath, isBundle } = parsed;
+        if (isBundle) {
+            return this.syncBundleCollection(name, spec, cwd, options);
+        }
+        onProgress?.(`Loading manifest from ${manifestPath}...`);
+        // Load and parse manifest based on protocol
+        let manifest;
+        let manifestContent;
+        let manifestDir;
+        if (protocol === 'file') {
+            manifest = await this.loadLocalManifest(manifestPath);
+            manifestContent = await readFile(manifestPath, 'utf8');
+            manifestDir = manifestPath.substring(0, manifestPath.lastIndexOf('/'));
+        }
+        else {
+            const result = await this.loadRemoteManifest(manifestPath);
+            manifest = result.manifest;
+            manifestContent = result.content;
+            manifestDir = this.getUrlDirectory(manifestPath);
+        }
+        // Check manifest hash to skip if unchanged
+        const manifestHash = createHash('sha256').update(manifestContent).digest('hex');
+        const existingCollection = await this.getCollection(collectionId);
+        if (!force && existingCollection?.manifest_hash === manifestHash) {
+            onProgress?.('Manifest unchanged, skipping sync');
+            return { added: 0, updated: 0, removed: 0, total: 0 };
+        }
+        onProgress?.('Resolving sources...');
+        // Resolve sources to file entries
+        const entries = await this.resolveManifestSources(manifest, manifestDir, protocol);
+        // Get existing documents from database
+        const documentsService = this.#services.get(DocumentsService);
+        const existingDocs = await documentsService.getDocumentIds(collectionId);
+        const existingMap = new Map(existingDocs.map((doc) => [doc.id, doc.hash]));
+        // Compute changes
+        const toAdd = [];
+        const toUpdate = [];
+        const toRemove = [];
+        for (const entry of entries) {
+            const existingHash = existingMap.get(entry.id);
+            if (!existingHash) {
+                toAdd.push(entry);
+            }
+            else if (force) {
+                toUpdate.push(entry);
+            }
+            else if (entry.hash) {
+                // Manifest provides hash, compare with stored hash
+                if (existingHash !== entry.hash) {
+                    toUpdate.push(entry);
+                }
+            }
+            else {
+                // No manifest hash, need to fetch and compare
+                toUpdate.push(entry);
+            }
+        }
+        const currentIds = new Set(entries.map((e) => e.id));
+        for (const [id] of existingMap) {
+            if (!currentIds.has(id)) {
+                toRemove.push(id);
+            }
+        }
+        // Apply changes
+        if (toRemove.length > 0) {
+            onProgress?.(`Removing ${toRemove.length} deleted documents...`);
+            await documentsService.deleteDocuments(collectionId, toRemove);
+        }
+        const toProcess = [...toAdd, ...toUpdate];
+        let actualUpdated = 0;
+        for (let i = 0; i < toProcess.length; i++) {
+            const entry = toProcess[i];
+            const isNew = toAdd.includes(entry);
+            onProgress?.(`${isNew ? 'Adding' : 'Checking'} ${entry.id} (${i + 1}/${toProcess.length})...`);
+            try {
+                const content = await this.fetchContent(entry.url);
+                const contentHash = createHash('sha256').update(content).digest('hex');
+                // Skip if content hash matches (for entries without manifest hash)
+                if (!isNew && !force && existingMap.get(entry.id) === contentHash) {
+                    continue;
+                }
+                if (!isNew)
+                    actualUpdated++;
+                await documentsService.updateDocument({
+                    collection: collectionId,
+                    id: entry.id,
+                    content,
+                });
+            }
+            catch (error) {
+                // Log warning but don't fail the entire sync for individual file failures
+                const errorMsg = error instanceof Error ? error.message : 'Unknown error';
+                onProgress?.(`Warning: Failed to fetch ${entry.id}: ${errorMsg}`);
+            }
+        }
+        // Update collection record
+        await this.upsertCollection(collectionId, {
+            url: spec.url,
+            name: manifest.name,
+            version: manifest.version,
+            description: manifest.description ?? null,
+            manifest_hash: manifestHash,
+            last_sync_at: new Date().toISOString(),
+        });
+        return {
+            added: toAdd.length,
+            updated: actualUpdated,
+            removed: toRemove.length,
+            total: entries.length,
+        };
+    };
+    /**
+     * Sync all collections from project config.
+     */
+    syncAllCollections = async (cwd = process.cwd(), options = {}) => {
+        const projectConfig = this.readProjectConfig(cwd);
+        const results = new Map();
+        for (const [name, spec] of Object.entries(projectConfig.collections)) {
+            const result = await this.syncCollection(name, spec, cwd, {
+                force: options.force,
+                onProgress: (message) => options.onProgress?.(name, message),
+            });
+            results.set(name, result);
+        }
+        return results;
+    };
+}
+export { CollectionsService };
+//# sourceMappingURL=collections.js.map