hazo_files 1.4.0 → 1.4.2

package/dist/server/index.js

@@ -45,6 +45,7 @@ __export(index_exports, {
   GoogleDriveAuth: () => GoogleDriveAuth,
   GoogleDriveModule: () => GoogleDriveModule,
   HAZO_FILES_DEFAULT_TABLE_NAME: () => HAZO_FILES_DEFAULT_TABLE_NAME,
+  HAZO_FILES_MIGRATION_V2: () => HAZO_FILES_MIGRATION_V2,
   HAZO_FILES_NAMING_DEFAULT_TABLE_NAME: () => HAZO_FILES_NAMING_DEFAULT_TABLE_NAME,
   HAZO_FILES_NAMING_TABLE_SCHEMA: () => HAZO_FILES_NAMING_TABLE_SCHEMA,
   HAZO_FILES_TABLE_SCHEMA: () => HAZO_FILES_TABLE_SCHEMA,
@@ -62,6 +63,8 @@ __export(index_exports, {
   TrackedFileManager: () => TrackedFileManager,
   UploadExtractService: () => UploadExtractService,
   addExtractionToFileData: () => addExtractionToFileData,
+  backfillV2Defaults: () => backfillV2Defaults,
+  buildFileWithStatus: () => buildFileWithStatus,
   clearExtractions: () => clearExtractions,
   clonePattern: () => clonePattern,
   computeFileHash: () => computeFileHash,
@@ -75,6 +78,7 @@ __export(index_exports, {
   createFileItem: () => createFileItem,
   createFileManager: () => createFileManager,
   createFileMetadataService: () => createFileMetadataService,
+  createFileRef: () => createFileRef,
   createFolderItem: () => createFolderItem,
   createGoogleDriveAuth: () => createGoogleDriveAuth,
   createGoogleDriveModule: () => createGoogleDriveModule,
@@ -98,6 +102,7 @@ __export(index_exports, {
   generateExtractionId: () => generateExtractionId,
   generateId: () => generateId,
   generatePreviewName: () => generatePreviewName,
+  generateRefId: () => generateRefId,
   generateSampleConfig: () => generateSampleConfig,
   generateSegmentId: () => generateSegmentId,
   getBaseName: () => getBaseName,
@@ -111,6 +116,7 @@ __export(index_exports, {
   getFileCategory: () => getFileCategory,
   getFileMetadataValues: () => getFileMetadataValues,
   getMergedData: () => getMergedData,
+  getMigrationForTable: () => getMigrationForTable,
   getMimeType: () => getMimeType,
   getNameWithoutExtension: () => getNameWithoutExtension,
   getNamingSchemaForTable: () => getNamingSchemaForTable,
@@ -142,20 +148,26 @@ __export(index_exports, {
   joinPath: () => joinPath,
   loadConfig: () => loadConfig,
   loadConfigAsync: () => loadConfigAsync,
+  migrateToV2: () => migrateToV2,
   normalizePath: () => normalizePath,
   parseConfig: () => parseConfig,
   parseFileData: () => parseFileData,
+  parseFileRefs: () => parseFileRefs,
   parsePatternString: () => parsePatternString,
   patternToString: () => patternToString,
   recalculateMergedData: () => recalculateMergedData,
   registerModule: () => registerModule,
   removeExtractionById: () => removeExtractionById,
   removeExtractionByIndex: () => removeExtractionByIndex,
+  removeRefFromArray: () => removeRefFromArray,
+  removeRefsByCriteriaFromArray: () => removeRefsByCriteriaFromArray,
   sanitizeFilename: () => sanitizeFilename,
   saveConfig: () => saveConfig,
   sortItems: () => sortItems,
   stringifyFileData: () => stringifyFileData,
+  stringifyFileRefs: () => stringifyFileRefs,
   successResult: () => successResult,
+  toV2Record: () => toV2Record,
   updateExtractionById: () => updateExtractionById,
   validateExtractionData: () => validateExtractionData,
   validateFileDataStructure: () => validateFileDataStructure,
@@ -780,7 +792,7 @@ var LocalStorageModule = class extends BaseStorageModule {
     await super.initialize(config);
     const localConfig = this.getProviderConfig();
     this.basePath = path2.resolve(localConfig.basePath);
-    this.allowedExtensions = localConfig.allowedExtensions || [];
+    this.allowedExtensions = (localConfig.allowedExtensions || []).map((ext) => ext.trim().replace(/^\./, "").toLowerCase()).filter((ext) => ext.length > 0);
     this.maxFileSize = localConfig.maxFileSize || 0;
     await fs2.promises.mkdir(this.basePath, { recursive: true });
   }
@@ -2271,6 +2283,70 @@ function updateExtractionById(fileData, id, newData, options = {}) {
   };
 }
 
+// src/common/ref-utils.ts
+function generateRefId() {
+  return `ref_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
+}
+function parseFileRefs(json) {
+  if (!json) return [];
+  try {
+    const parsed = JSON.parse(json);
+    if (!Array.isArray(parsed)) return [];
+    return parsed;
+  } catch {
+    return [];
+  }
+}
+function stringifyFileRefs(refs) {
+  return JSON.stringify(refs);
+}
+function createFileRef(options) {
+  const ref = {
+    ref_id: generateRefId(),
+    entity_type: options.entity_type,
+    entity_id: options.entity_id,
+    created_at: (/* @__PURE__ */ new Date()).toISOString()
+  };
+  if (options.created_by) ref.created_by = options.created_by;
+  if (options.visibility) ref.visibility = options.visibility;
+  if (options.label) ref.label = options.label;
+  if (options.metadata) ref.metadata = options.metadata;
+  return ref;
+}
+function removeRefFromArray(refs, refId) {
+  return refs.filter((r) => r.ref_id !== refId);
+}
+function removeRefsByCriteriaFromArray(refs, criteria) {
+  return refs.filter((ref) => {
+    if (criteria.entity_type && ref.entity_type !== criteria.entity_type) return true;
+    if (criteria.entity_id && ref.entity_id !== criteria.entity_id) return true;
+    return !criteria.entity_type && !criteria.entity_id;
+  });
+}
+function toV2Record(record) {
+  const v2 = record;
+  return {
+    ...record,
+    file_refs: typeof v2.file_refs === "string" ? v2.file_refs : "[]",
+    ref_count: typeof v2.ref_count === "number" ? v2.ref_count : 0,
+    status: typeof v2.status === "string" ? v2.status : "active",
+    scope_id: typeof v2.scope_id === "string" ? v2.scope_id : null,
+    uploaded_by: typeof v2.uploaded_by === "string" ? v2.uploaded_by : null,
+    original_filename: typeof v2.original_filename === "string" ? v2.original_filename : null,
+    storage_verified_at: typeof v2.storage_verified_at === "string" ? v2.storage_verified_at : null,
+    deleted_at: typeof v2.deleted_at === "string" ? v2.deleted_at : null
+  };
+}
+function buildFileWithStatus(record) {
+  const v2 = toV2Record(record);
+  const refs = parseFileRefs(v2.file_refs);
+  return {
+    record: v2,
+    refs,
+    is_orphaned: refs.length === 0
+  };
+}
+
 // src/services/file-metadata-service.ts
 var FileMetadataService = class {
   constructor(crudService, options = {}) {
@@ -2312,9 +2388,15 @@ var FileMetadataService = class {
         changed_at: timestamp,
         file_hash: input.file_hash || null,
         file_size: input.file_size ?? null,
-        file_changed_at: input.file_hash ? timestamp : null
-        // Set content changed time if hash is provided
+        file_changed_at: input.file_hash ? timestamp : null,
+        // V2 defaults — included conditionally to avoid breaking pre-migration DBs
+        file_refs: "[]",
+        ref_count: 0,
+        status: "active"
       };
+      if (input.scope_id !== void 0) record.scope_id = input.scope_id;
+      if (input.uploaded_by !== void 0) record.uploaded_by = input.uploaded_by;
+      if (input.original_filename !== void 0) record.original_filename = input.original_filename;
       const results = await this.crud.insert(record);
       this.logger?.debug?.("Recorded file upload", { path: input.file_path });
       return results[0] || null;
@@ -2673,6 +2755,254 @@ var FileMetadataService = class {
       return false;
     }
   }
+  // ============================================
+  // Reference Tracking Methods (V2)
+  // ============================================
+  /**
+   * Find a record by ID
+   */
+  async findById(id) {
+    try {
+      const results = await this.crud.findBy({ id });
+      return results[0] || null;
+    } catch (error) {
+      this.logError("findById", error);
+      return null;
+    }
+  }
+  /**
+   * Find multiple records by IDs
+   */
+  async findByIds(ids) {
+    try {
+      const results = [];
+      for (const id of ids) {
+        const record = await this.findById(id);
+        if (record) results.push(record);
+      }
+      return results;
+    } catch (error) {
+      this.logError("findByIds", error);
+      return [];
+    }
+  }
+  /**
+   * Add a reference to a file
+   * @returns The new ref_id, or null on failure
+   */
+  async addRef(fileId, options) {
+    try {
+      const record = await this.findById(fileId);
+      if (!record) {
+        this.logger?.warn?.("Cannot add ref: file not found", { fileId });
+        return null;
+      }
+      const v2 = toV2Record(record);
+      const refs = parseFileRefs(v2.file_refs);
+      const newRef = createFileRef(options);
+      const updatedRefs = [...refs, newRef];
+      await this.crud.updateById(fileId, {
+        file_refs: stringifyFileRefs(updatedRefs),
+        ref_count: updatedRefs.length,
+        status: "active",
+        changed_at: this.now()
+      });
+      this.logger?.debug?.("Added ref", { fileId, ref_id: newRef.ref_id });
+      return { ref_id: newRef.ref_id };
+    } catch (error) {
+      this.logError("addRef", error);
+      return null;
+    }
+  }
+  /**
+   * Remove a specific reference from a file
+   * @returns Remaining ref count, or null on failure
+   */
+  async removeRef(fileId, refId) {
+    try {
+      const record = await this.findById(fileId);
+      if (!record) {
+        this.logger?.warn?.("Cannot remove ref: file not found", { fileId });
+        return null;
+      }
+      const v2 = toV2Record(record);
+      const refs = parseFileRefs(v2.file_refs);
+      const updatedRefs = removeRefFromArray(refs, refId);
+      await this.crud.updateById(fileId, {
+        file_refs: stringifyFileRefs(updatedRefs),
+        ref_count: updatedRefs.length,
+        changed_at: this.now()
+      });
+      this.logger?.debug?.("Removed ref", { fileId, refId, remaining: updatedRefs.length });
+      return { remaining_refs: updatedRefs.length };
+    } catch (error) {
+      this.logError("removeRef", error);
+      return null;
+    }
+  }
+  /**
+   * Remove references matching criteria across all records.
+   * Scans all records and removes matching refs (AND semantics).
+   */
+  async removeRefsByCriteria(criteria) {
+    try {
+      let totalRemoved = 0;
+      if (criteria.file_id) {
+        const record = await this.findById(criteria.file_id);
+        if (record) {
+          const removed = await this.removeRefsFromRecord(record, criteria);
+          totalRemoved += removed;
+        }
+        return { removed_count: totalRemoved };
+      }
+      let records;
+      if (criteria.scope_id) {
+        records = await this.crud.findBy({ scope_id: criteria.scope_id });
+      } else {
+        records = await this.crud.list();
+      }
+      for (const record of records) {
+        const removed = await this.removeRefsFromRecord(record, criteria);
+        totalRemoved += removed;
+      }
+      this.logger?.debug?.("Removed refs by criteria", { criteria, removed_count: totalRemoved });
+      return { removed_count: totalRemoved };
+    } catch (error) {
+      this.logError("removeRefsByCriteria", error);
+      return { removed_count: 0 };
+    }
+  }
+  /**
+   * Helper: remove matching refs from a single record
+   */
+  async removeRefsFromRecord(record, criteria) {
+    const v2 = toV2Record(record);
+    const refs = parseFileRefs(v2.file_refs);
+    if (refs.length === 0) return 0;
+    const updatedRefs = removeRefsByCriteriaFromArray(refs, {
+      entity_type: criteria.entity_type,
+      entity_id: criteria.entity_id
+    });
+    const removedCount = refs.length - updatedRefs.length;
+    if (removedCount > 0) {
+      await this.crud.updateById(record.id, {
+        file_refs: stringifyFileRefs(updatedRefs),
+        ref_count: updatedRefs.length,
+        changed_at: this.now()
+      });
+    }
+    return removedCount;
+  }
+  /**
+   * Get all references for a file
+   */
+  async getRefs(fileId) {
+    try {
+      const record = await this.findById(fileId);
+      if (!record) return null;
+      return parseFileRefs(toV2Record(record).file_refs);
+    } catch (error) {
+      this.logError("getRefs", error);
+      return null;
+    }
+  }
+  /**
+   * Get a file with its status and parsed refs
+   */
+  async getFileWithStatus(fileId) {
+    try {
+      const record = await this.findById(fileId);
+      if (!record) return null;
+      return buildFileWithStatus(record);
+    } catch (error) {
+      this.logError("getFileWithStatus", error);
+      return null;
+    }
+  }
+  /**
+   * Get multiple files with status
+   */
+  async getFilesWithStatus(fileIds) {
+    try {
+      const records = await this.findByIds(fileIds);
+      return records.map(buildFileWithStatus);
+    } catch (error) {
+      this.logError("getFilesWithStatus", error);
+      return [];
+    }
+  }
+  /**
+   * Update the status of a file
+   */
+  async updateStatus(fileId, status) {
+    try {
+      const patch = {
+        status,
+        changed_at: this.now()
+      };
+      if (status === "soft_deleted") {
+        patch.deleted_at = this.now();
+      }
+      await this.crud.updateById(fileId, patch);
+      this.logger?.debug?.("Updated status", { fileId, status });
+      return true;
+    } catch (error) {
+      this.logError("updateStatus", error);
+      return false;
+    }
+  }
+  /**
+   * Soft-delete a file (set status to soft_deleted, record deleted_at)
+   */
+  async softDelete(fileId) {
+    return this.updateStatus(fileId, "soft_deleted");
+  }
+  /**
+   * Update specific V2 fields on a record
+   */
+  async updateFields(fileId, fields) {
+    try {
+      await this.crud.updateById(fileId, {
+        ...fields,
+        changed_at: this.now()
+      });
+      this.logger?.debug?.("Updated fields", { fileId, fields: Object.keys(fields) });
+      return true;
+    } catch (error) {
+      this.logError("updateFields", error);
+      return false;
+    }
+  }
+  /**
+   * Find orphaned files (zero references)
+   */
+  async findOrphaned(options) {
+    try {
+      let records;
+      if (options?.scope_id) {
+        records = await this.crud.findBy({ scope_id: options.scope_id });
+      } else if (options?.storage_type) {
+        records = await this.crud.findBy({ storage_type: options.storage_type });
+      } else {
+        records = await this.crud.list();
+      }
+      let orphaned = records.map(buildFileWithStatus).filter((f) => f.is_orphaned && f.record.status !== "soft_deleted");
+      if (options?.olderThanMs) {
+        const cutoff = Date.now() - options.olderThanMs;
+        orphaned = orphaned.filter((f) => {
+          const createdAt = new Date(f.record.created_at).getTime();
+          return createdAt < cutoff;
+        });
+      }
+      if (options?.limit && orphaned.length > options.limit) {
+        orphaned = orphaned.slice(0, options.limit);
+      }
+      return orphaned;
+    } catch (error) {
+      this.logError("findOrphaned", error);
+      return [];
+    }
+  }
 };
 function createFileMetadataService(crudService, options) {
   return new FileMetadataService(crudService, options);
@@ -3042,6 +3372,132 @@ var TrackedFileManager = class extends FileManager {
     const record = await this.metadataService.findByPath(path3, this.getStorageType());
     return record?.file_size ?? null;
   }
+  // ============ Reference Tracking Methods (V2) ============
+  /**
+   * Add a reference to a file
+   */
+  async addRef(fileId, options) {
+    if (!this.isTrackingEnabled()) return null;
+    return this.metadataService.addRef(fileId, options);
+  }
+  /**
+   * Remove a reference from a file
+   */
+  async removeRef(fileId, refId) {
+    if (!this.isTrackingEnabled()) return null;
+    return this.metadataService.removeRef(fileId, refId);
+  }
+  /**
+   * Get a file by its database ID with status information
+   */
+  async getFileById(fileId) {
+    if (!this.isTrackingEnabled()) return null;
+    return this.metadataService.getFileWithStatus(fileId);
+  }
+  /**
+   * Get multiple files by their database IDs with status information
+   */
+  async getFilesById(fileIds) {
+    if (!this.isTrackingEnabled()) return [];
+    return this.metadataService.getFilesWithStatus(fileIds);
+  }
+  /**
+   * Soft-delete a file (marks as soft_deleted, does not remove physical file)
+   */
+  async softDeleteFile(fileId) {
+    if (!this.isTrackingEnabled()) return false;
+    return this.metadataService.softDelete(fileId);
+  }
+  /**
+   * Find orphaned files (files with zero references)
+   */
+  async findOrphanedFiles(options) {
+    if (!this.isTrackingEnabled()) return [];
+    return this.metadataService.findOrphaned(options);
+  }
+  /**
+   * Cleanup orphaned files — removes physical files and/or DB records
+   */
+  async cleanupOrphanedFiles(options) {
+    if (!this.isTrackingEnabled()) return { cleaned: 0, errors: [] };
+    const orphaned = await this.metadataService.findOrphaned(options);
+    let cleaned = 0;
+    const errors = [];
+    for (const file of orphaned) {
+      try {
+        if (options?.softDeleteOnly) {
+          await this.metadataService.softDelete(file.record.id);
+          cleaned++;
+          continue;
+        }
+        const deletePhysical = options?.deletePhysicalFiles !== false;
+        if (deletePhysical) {
+          const deleteResult = await super.deleteFile(file.record.file_path);
+          if (!deleteResult.success) {
+            if (deleteResult.error && !deleteResult.error.includes("not found")) {
+              errors.push(`Failed to delete physical file ${file.record.file_path}: ${deleteResult.error}`);
+            }
+          }
+        }
+        await this.metadataService.recordDelete(file.record.file_path, file.record.storage_type);
+        cleaned++;
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        errors.push(`Error cleaning up ${file.record.file_path}: ${msg}`);
+      }
+    }
+    return { cleaned, errors };
+  }
+  /**
+   * Verify that a file's physical storage exists and update its status
+   */
+  async verifyFileExistence(fileId) {
+    if (!this.isTrackingEnabled()) return null;
+    const record = await this.metadataService.findById(fileId);
+    if (!record) return null;
+    const fileExists = await this.exists(record.file_path);
+    const timestamp = (/* @__PURE__ */ new Date()).toISOString();
+    await this.metadataService.updateFields(fileId, {
+      storage_verified_at: timestamp,
+      ...fileExists ? {} : { status: "missing" }
+    });
+    return fileExists;
+  }
+  /**
+   * Upload a file and optionally add an initial reference
+   */
+  async uploadFileWithRef(source, remotePath, options) {
+    const uploadResult = await this.uploadFile(source, remotePath, {
+      ...options,
+      awaitRecording: true
+    });
+    if (!uploadResult.success || !uploadResult.data || !this.isTrackingEnabled()) {
+      return uploadResult;
+    }
+    const record = await this.metadataService.findByPath(remotePath, this.getStorageType());
+    if (!record) {
+      return uploadResult;
+    }
+    const fieldsToUpdate = {};
+    if (options?.scope_id) fieldsToUpdate.scope_id = options.scope_id;
+    if (options?.uploaded_by) fieldsToUpdate.uploaded_by = options.uploaded_by;
+    if (Object.keys(fieldsToUpdate).length > 0) {
+      await this.metadataService.updateFields(record.id, fieldsToUpdate);
+    }
+    let refId;
+    if (options?.ref) {
+      const refResult = await this.metadataService.addRef(record.id, options.ref);
+      if (refResult) refId = refResult.ref_id;
+    }
+    return {
+      success: true,
+      data: {
+        ...uploadResult.data,
+        file_id: record.id,
+        ref_id: refId
+      }
+    };
+  }
 };
 function createTrackedFileManager(options) {
   return new TrackedFileManager(options);
@@ -4090,13 +4546,25 @@ var HAZO_FILES_TABLE_SCHEMA = {
   storage_type TEXT NOT NULL,
   file_hash TEXT,
   file_size INTEGER,
-  file_changed_at TEXT
+  file_changed_at TEXT,
+  file_refs TEXT DEFAULT '[]',
+  ref_count INTEGER DEFAULT 0,
+  status TEXT DEFAULT 'active',
+  scope_id TEXT,
+  uploaded_by TEXT,
+  storage_verified_at TEXT,
+  deleted_at TEXT,
+  original_filename TEXT
 )`,
     indexes: [
       "CREATE INDEX IF NOT EXISTS idx_hazo_files_path ON hazo_files (file_path)",
       "CREATE INDEX IF NOT EXISTS idx_hazo_files_storage ON hazo_files (storage_type)",
       "CREATE UNIQUE INDEX IF NOT EXISTS idx_hazo_files_path_storage ON hazo_files (file_path, storage_type)",
-      "CREATE INDEX IF NOT EXISTS idx_hazo_files_hash ON hazo_files (file_hash)"
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_hash ON hazo_files (file_hash)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_status ON hazo_files (status)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_scope ON hazo_files (scope_id)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_ref_count ON hazo_files (ref_count)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_deleted ON hazo_files (deleted_at)"
     ]
   },
   postgres: {
@@ -4111,13 +4579,25 @@ var HAZO_FILES_TABLE_SCHEMA = {
   storage_type TEXT NOT NULL,
   file_hash TEXT,
   file_size BIGINT,
-  file_changed_at TIMESTAMP WITH TIME ZONE
+  file_changed_at TIMESTAMP WITH TIME ZONE,
+  file_refs TEXT DEFAULT '[]',
+  ref_count INTEGER DEFAULT 0,
+  status TEXT DEFAULT 'active',
+  scope_id UUID,
+  uploaded_by UUID,
+  storage_verified_at TIMESTAMP WITH TIME ZONE,
+  deleted_at TIMESTAMP WITH TIME ZONE,
+  original_filename TEXT
 )`,
     indexes: [
       "CREATE INDEX IF NOT EXISTS idx_hazo_files_path ON hazo_files (file_path)",
       "CREATE INDEX IF NOT EXISTS idx_hazo_files_storage ON hazo_files (storage_type)",
       "CREATE UNIQUE INDEX IF NOT EXISTS idx_hazo_files_path_storage ON hazo_files (file_path, storage_type)",
-      "CREATE INDEX IF NOT EXISTS idx_hazo_files_hash ON hazo_files (file_hash)"
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_hash ON hazo_files (file_hash)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_status ON hazo_files (status)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_scope ON hazo_files (scope_id)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_ref_count ON hazo_files (ref_count)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_deleted ON hazo_files (deleted_at)"
     ]
   },
   columns: [
@@ -4131,7 +4611,15 @@ var HAZO_FILES_TABLE_SCHEMA = {
     "storage_type",
     "file_hash",
     "file_size",
-    "file_changed_at"
+    "file_changed_at",
+    "file_refs",
+    "ref_count",
+    "status",
+    "scope_id",
+    "uploaded_by",
+    "storage_verified_at",
+    "deleted_at",
+    "original_filename"
   ]
 };
 function getSchemaForTable(tableName, dbType) {
@@ -4144,6 +4632,78 @@ function getSchemaForTable(tableName, dbType) {
     )
   };
 }
+var HAZO_FILES_MIGRATION_V2 = {
+  tableName: HAZO_FILES_DEFAULT_TABLE_NAME,
+  sqlite: {
+    alterStatements: [
+      "ALTER TABLE hazo_files ADD COLUMN file_refs TEXT DEFAULT '[]'",
+      "ALTER TABLE hazo_files ADD COLUMN ref_count INTEGER DEFAULT 0",
+      "ALTER TABLE hazo_files ADD COLUMN status TEXT DEFAULT 'active'",
+      "ALTER TABLE hazo_files ADD COLUMN scope_id TEXT",
+      "ALTER TABLE hazo_files ADD COLUMN uploaded_by TEXT",
+      "ALTER TABLE hazo_files ADD COLUMN storage_verified_at TEXT",
+      "ALTER TABLE hazo_files ADD COLUMN deleted_at TEXT",
+      "ALTER TABLE hazo_files ADD COLUMN original_filename TEXT"
+    ],
+    indexes: [
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_status ON hazo_files (status)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_scope ON hazo_files (scope_id)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_ref_count ON hazo_files (ref_count)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_deleted ON hazo_files (deleted_at)"
+    ],
+    backfill: `UPDATE hazo_files SET
+  file_refs = COALESCE(file_refs, '[]'),
+  ref_count = COALESCE(ref_count, 0),
+  status = COALESCE(status, 'active')
+WHERE file_refs IS NULL OR ref_count IS NULL OR status IS NULL`
+  },
+  postgres: {
+    alterStatements: [
+      "ALTER TABLE hazo_files ADD COLUMN IF NOT EXISTS file_refs TEXT DEFAULT '[]'",
+      "ALTER TABLE hazo_files ADD COLUMN IF NOT EXISTS ref_count INTEGER DEFAULT 0",
+      "ALTER TABLE hazo_files ADD COLUMN IF NOT EXISTS status TEXT DEFAULT 'active'",
+      "ALTER TABLE hazo_files ADD COLUMN IF NOT EXISTS scope_id UUID",
+      "ALTER TABLE hazo_files ADD COLUMN IF NOT EXISTS uploaded_by UUID",
+      "ALTER TABLE hazo_files ADD COLUMN IF NOT EXISTS storage_verified_at TIMESTAMP WITH TIME ZONE",
+      "ALTER TABLE hazo_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMP WITH TIME ZONE",
+      "ALTER TABLE hazo_files ADD COLUMN IF NOT EXISTS original_filename TEXT"
+    ],
+    indexes: [
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_status ON hazo_files (status)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_scope ON hazo_files (scope_id)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_ref_count ON hazo_files (ref_count)",
+      "CREATE INDEX IF NOT EXISTS idx_hazo_files_deleted ON hazo_files (deleted_at)"
+    ],
+    backfill: `UPDATE hazo_files SET
+  file_refs = COALESCE(file_refs, '[]'),
+  ref_count = COALESCE(ref_count, 0),
+  status = COALESCE(status, 'active')
+WHERE file_refs IS NULL OR ref_count IS NULL OR status IS NULL`
+  },
+  newColumns: [
+    "file_refs",
+    "ref_count",
+    "status",
+    "scope_id",
+    "uploaded_by",
+    "storage_verified_at",
+    "deleted_at",
+    "original_filename"
+  ]
+};
+function getMigrationForTable(tableName, dbType) {
+  const migration = HAZO_FILES_MIGRATION_V2[dbType];
+  const defaultName = HAZO_FILES_MIGRATION_V2.tableName;
+  return {
+    alterStatements: migration.alterStatements.map(
+      (stmt) => stmt.replace(new RegExp(defaultName, "g"), tableName)
+    ),
+    indexes: migration.indexes.map(
+      (idx) => idx.replace(new RegExp(defaultName, "g"), tableName)
+    ),
+    backfill: migration.backfill.replace(new RegExp(defaultName, "g"), tableName)
+  };
+}
 var HAZO_FILES_NAMING_DEFAULT_TABLE_NAME = "hazo_files_naming";
 var HAZO_FILES_NAMING_TABLE_SCHEMA = {
   tableName: HAZO_FILES_NAMING_DEFAULT_TABLE_NAME,
@@ -4201,6 +4761,24 @@ function getNamingSchemaForTable(tableName, dbType) {
   };
 }
 
+// src/migrations/add-reference-tracking.ts
+async function migrateToV2(executor, dbType, tableName) {
+  const migration = tableName ? getMigrationForTable(tableName, dbType) : HAZO_FILES_MIGRATION_V2[dbType];
+  for (const stmt of migration.alterStatements) {
+    try {
+      await executor.run(stmt);
+    } catch {
+    }
+  }
+  for (const idx of migration.indexes) {
+    await executor.run(idx);
+  }
+}
+async function backfillV2Defaults(executor, dbType, tableName) {
+  const migration = tableName ? getMigrationForTable(tableName, dbType) : HAZO_FILES_MIGRATION_V2[dbType];
+  await executor.run(migration.backfill);
+}
+
 // src/server/index.ts
 try {
   require("server-only");
@@ -4223,6 +4801,7 @@ try {
   GoogleDriveAuth,
   GoogleDriveModule,
   HAZO_FILES_DEFAULT_TABLE_NAME,
+  HAZO_FILES_MIGRATION_V2,
   HAZO_FILES_NAMING_DEFAULT_TABLE_NAME,
   HAZO_FILES_NAMING_TABLE_SCHEMA,
   HAZO_FILES_TABLE_SCHEMA,
@@ -4240,6 +4819,8 @@ try {
   TrackedFileManager,
   UploadExtractService,
   addExtractionToFileData,
+  backfillV2Defaults,
+  buildFileWithStatus,
   clearExtractions,
   clonePattern,
   computeFileHash,
@@ -4253,6 +4834,7 @@ try {
   createFileItem,
   createFileManager,
   createFileMetadataService,
+  createFileRef,
   createFolderItem,
   createGoogleDriveAuth,
   createGoogleDriveModule,
@@ -4276,6 +4858,7 @@ try {
   generateExtractionId,
   generateId,
   generatePreviewName,
+  generateRefId,
   generateSampleConfig,
   generateSegmentId,
   getBaseName,
@@ -4289,6 +4872,7 @@ try {
   getFileCategory,
   getFileMetadataValues,
   getMergedData,
+  getMigrationForTable,
   getMimeType,
   getNameWithoutExtension,
   getNamingSchemaForTable,
@@ -4320,20 +4904,26 @@ try {
   joinPath,
   loadConfig,
   loadConfigAsync,
+  migrateToV2,
   normalizePath,
   parseConfig,
   parseFileData,
+  parseFileRefs,
   parsePatternString,
   patternToString,
   recalculateMergedData,
   registerModule,
   removeExtractionById,
   removeExtractionByIndex,
+  removeRefFromArray,
+  removeRefsByCriteriaFromArray,
   sanitizeFilename,
   saveConfig,
   sortItems,
   stringifyFileData,
+  stringifyFileRefs,
   successResult,
+  toV2Record,
   updateExtractionById,
   validateExtractionData,
   validateFileDataStructure,