@byline/db-mysql 6.11.0 → 6.12.1

package/dist/database/schema/index.d.ts

@@ -3163,6 +3163,40 @@ export declare const fileStore: import("drizzle-orm/mysql-core").MySqlTableWithC
             identity: undefined;
             generated: undefined;
         }, {}, {}>;
+        storage_validator_kind: import("drizzle-orm/mysql-core").MySqlColumn<{
+            name: "storage_validator_kind";
+            tableName: "byline_store_file";
+            dataType: "string";
+            columnType: "MySqlVarChar";
+            data: string;
+            driverParam: string | number;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
+        storage_validator_value: import("drizzle-orm/mysql-core").MySqlColumn<{
+            name: "storage_validator_value";
+            tableName: "byline_store_file";
+            dataType: "string";
+            columnType: "MySqlVarChar";
+            data: string;
+            driverParam: string | number;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
         storage_provider: import("drizzle-orm/mysql-core").MySqlColumn<{
             name: "storage_provider";
             tableName: "byline_store_file";

package/dist/database/schema/index.js

@@ -678,7 +678,11 @@ export const fileStore = mysqlTable('byline_store_file', {
     // File metadata
     mime_type: varchar('mime_type', { length: 100 }).notNull(),
     file_size: bigint('file_size', { mode: 'number' }).notNull(), // Size in bytes
-    file_hash: varchar('file_hash', { length: 64 }), // SHA-256 hash for deduplication
+    file_hash: varchar('file_hash', { length: 64 }), // SHA-256 content hash (lower-case hex)
+    // Storage validator of the stored version that file_hash was computed
+    // from (S3 version id / ETag, local stat). Written and read as a pair.
+    storage_validator_kind: varchar('storage_validator_kind', { length: 32 }),
+    storage_validator_value: varchar('storage_validator_value', { length: 1024 }),
     // Storage information
     storage_provider: varchar('storage_provider', { length: 50 }).notNull(), // 'local', 's3', 'gcs', etc.
     storage_path: text('storage_path').notNull(),

package/dist/index.js

@@ -20,6 +20,7 @@ import { createReadSnapshot } from './modules/storage/read-snapshot.js';
 import { SingletonCommands, SingletonQueries } from './modules/storage/singletons.js';
 import { createCommandBuilders } from './modules/storage/storage-commands.js';
 import { createQueryBuilders } from './modules/storage/storage-queries.js';
+import { StoredFileCommands, StoredFileQueries } from './modules/storage/stored-files.js';
 export const mysqlAdapter = ({ connectionString, collections, defaultContentLocale, connectionLimit = 20, idleTimeout = 2000, connectTimeout = 30000, maxIdle, }) => {
     const pool = mysql.createPool({
         uri: connectionString,
@@ -139,6 +140,7 @@ export const mysqlAdapter = ({ connectionString, collections, defaultContentLoca
             // see `./modules/audit/audit-commands.js`.
             audit: auditCommands,
             singletons: singletonCommands,
+            storedFiles: new StoredFileCommands(dbManager),
         },
         queries: {
             // `queryBuilders.collections` fully implements `ICollectionQueries`
@@ -151,6 +153,7 @@ export const mysqlAdapter = ({ connectionString, collections, defaultContentLoca
             // `./modules/audit/audit-queries.js`.
             audit: auditQueries,
             singletons: singletonQueries,
+            storedFiles: new StoredFileQueries(dbManager),
         },
         withTransaction: (fn) => txManager.withTransaction(fn),
         withReadSnapshot: createReadSnapshot(db, collections, defaultContentLocale),

package/dist/modules/storage/storage-insert.js

@@ -102,6 +102,8 @@ export const prepareFieldInsertBuckets = (flattenedFields, document_version_id,
                     storage_path: field.storage_path,
                     storage_url: field.storage_url,
                     file_hash: field.file_hash,
+                    storage_validator_kind: field.storage_validator_kind,
+                    storage_validator_value: field.storage_validator_value,
                     image_width: field.image_width,
                     image_height: field.image_height,
                     image_format: field.image_format,

package/dist/modules/storage/storage-queries.js

@@ -2056,6 +2056,8 @@ export class DocumentQueries {
                         storage_path: row.storage_path,
                         storage_url: row.storage_url,
                         file_hash: row.file_hash,
+                        storage_validator_kind: row.storage_validator_kind,
+                        storage_validator_value: row.storage_validator_value,
                         image_width: row.image_width,
                         image_height: row.image_height,
                         image_format: row.image_format,

package/dist/modules/storage/stored-files.d.ts

@@ -0,0 +1,27 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import type { IStoredFileCommands, IStoredFileQueries, ListStoredFilePathsParams, StorageValidator, StoredFilePathGroup } from '@byline/core';
+import type { DBManager } from '../../lib/db-manager.js';
+export declare class StoredFileQueries implements IStoredFileQueries {
+    private dbManager;
+    constructor(dbManager: DBManager);
+    private get db();
+    listStoredFilePaths(params: ListStoredFilePathsParams): Promise<StoredFilePathGroup[]>;
+}
+export declare class StoredFileCommands implements IStoredFileCommands {
+    private dbManager;
+    constructor(dbManager: DBManager);
+    private get db();
+    setIdentityForPath(params: {
+        storageProvider: string;
+        storagePath: string;
+        collectionId?: string;
+        fileHash: string;
+        storageValidator: StorageValidator;
+    }): Promise<number>;
+}

package/dist/modules/storage/stored-files.js

@@ -0,0 +1,111 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { sql } from 'drizzle-orm';
+// Stored-file identity maintenance (backfill / verify). Storage keys,
+// provider names and recorded digests must compare exactly. The database
+// default collation (utf8mb4_0900_ai_ci) folds case and accents, and even
+// utf8mb4_bin is PAD SPACE (it equates 'a.pdf' and 'a.pdf '). utf8mb4_0900_bin
+// is NO PAD but needs MySQL 8.0.17, above the adapter's 8.0.14 floor. So every
+// comparison, DISTINCT, ORDER BY, join and GROUP BY here goes through
+// CAST(... AS BINARY): byte equality, trailing spaces significant, byte order.
+const binary = (expression) => sql `CAST(${expression} AS BINARY)`;
+function scopeFilter(params, alias = '') {
+    const column = (name) => sql.raw(`${alias}${name}`);
+    const parts = [
+        sql `${binary(column('storage_provider'))} = ${binary(sql `${params.storageProvider}`)}`,
+        params.identity === 'missing'
+            ? sql `${column('file_hash')} IS NULL`
+            : sql `${column('file_hash')} IS NOT NULL`,
+    ];
+    if (params.collectionId !== undefined) {
+        parts.push(sql `${column('collection_id')} = ${params.collectionId}`);
+    }
+    return sql.join(parts, sql ` AND `);
+}
+function rowsOf(result) {
+    return result[0];
+}
+/** mysql2 returns binary-cast columns as Buffers; decode them back to text. */
+function text(value) {
+    return Buffer.isBuffer(value) ? value.toString('utf8') : value;
+}
+export class StoredFileQueries {
+    dbManager;
+    constructor(dbManager) {
+        this.dbManager = dbManager;
+    }
+    get db() {
+        return this.dbManager.get();
+    }
+    async listStoredFilePaths(params) {
+        const after = params.afterPath === undefined
+            ? sql ``
+            : sql ` AND ${binary(sql `storage_path`)} > ${binary(sql `${params.afterPath}`)}`;
+        // A page is a bounded set of distinct paths; `limit` never counts rows.
+        const pathsQuery = sql `
+      SELECT DISTINCT ${binary(sql `storage_path`)} AS path_key
+      FROM byline_store_file
+      WHERE ${scopeFilter(params)}${after}
+      ORDER BY path_key
+      LIMIT ${params.limit}
+    `;
+        if (params.identity === 'missing') {
+            const result = await this.db.execute(pathsQuery);
+            return rowsOf(result).map((row) => ({
+                storagePath: text(row.path_key),
+                hashes: [],
+            }));
+        }
+        // Every recorded-hash group for each selected path, so a path's groups are
+        // never split across pages. Digests group by bytes, so differently spelled
+        // legacy values are reported separately.
+        const result = await this.db.execute(sql `
+      WITH paths AS (${pathsQuery})
+      SELECT p.path_key, ${binary(sql `f.file_hash`)} AS hash_key, COUNT(*) AS row_count
+      FROM byline_store_file f
+      JOIN paths p ON ${binary(sql `f.storage_path`)} = p.path_key
+      WHERE ${scopeFilter(params, 'f.')}
+      GROUP BY p.path_key, hash_key
+      ORDER BY p.path_key, hash_key
+    `);
+        const groups = [];
+        for (const row of rowsOf(result)) {
+            const storagePath = text(row.path_key);
+            let group = groups.at(-1);
+            if (!group || group.storagePath !== storagePath) {
+                group = { storagePath, hashes: [] };
+                groups.push(group);
+            }
+            group.hashes.push({ fileHash: text(row.hash_key), rowCount: Number(row.row_count) });
+        }
+        return groups;
+    }
+}
+export class StoredFileCommands {
+    dbManager;
+    constructor(dbManager) {
+        this.dbManager = dbManager;
+    }
+    get db() {
+        return this.dbManager.get();
+    }
+    async setIdentityForPath(params) {
+        const collection = params.collectionId === undefined ? sql `` : sql ` AND collection_id = ${params.collectionId}`;
+        const result = await this.db.execute(sql `
+      UPDATE byline_store_file
+      SET file_hash = ${params.fileHash},
+          storage_validator_kind = ${params.storageValidator.kind},
+          storage_validator_value = ${params.storageValidator.value},
+          updated_at = NOW()
+      WHERE ${binary(sql `storage_provider`)} = ${binary(sql `${params.storageProvider}`)}
+        AND ${binary(sql `storage_path`)} = ${binary(sql `${params.storagePath}`)}
+        AND file_hash IS NULL${collection}
+    `);
+        return result[0].affectedRows;
+    }
+}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@byline/db-mysql",
   "private": false,
   "license": "MPL-2.0",
-  "version": "6.11.0",
+  "version": "6.12.1",
   "engines": {
     "node": ">=22.22.2"
   },
@@ -47,24 +47,24 @@
     "dist"
   ],
   "dependencies": {
-    "@byline/admin": "6.11.0",
-    "@byline/core": "6.11.0",
+    "@byline/admin": "6.12.1",
+    "@byline/core": "6.12.1",
     "drizzle-orm": "^0.45.4",
     "mysql2": "^3.24.6",
     "uuid": "^14.0.3"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.6.2",
-    "@byline/auth": "6.11.0",
-    "@byline/client": "6.11.0",
+    "@biomejs/biome": "2.6.4",
+    "@byline/auth": "6.12.1",
+    "@byline/client": "6.12.1",
     "@byline/db-conformance": "0.0.10",
-    "@types/node": "^26.6.5",
+    "@types/node": "^26.6.6",
     "chokidar": "^6.0.0",
     "chokidar-cli": "^3.0.0",
-    "dotenv": "^18.0.8",
+    "dotenv": "^18.0.9",
     "drizzle-kit": "^0.31.11",
     "npm-run-all": "^4.1.5",
-    "tsc-alias": "^1.9.8",
+    "tsc-alias": "^1.9.10",
     "typescript": "^7.0.2",
     "vitest": "^5.0.5"
   },
@@ -80,6 +80,8 @@
     "db:init": "cd src/database && ./db_init.sh",
     "drizzle:generate": "tsc -p tsconfig.json && drizzle-kit generate --config=drizzle.config.ts",
     "drizzle:migrate": "drizzle-kit migrate --config=drizzle.config.ts",
+    "drizzle:rekey": "node --env-file-if-exists=.env scripts/drizzle-rekey.mjs",
+    "drizzle:rekey:test": "node --env-file-if-exists=.env.test scripts/drizzle-rekey.mjs",
     "drizzle:push": "tsc -p tsconfig.json && drizzle-kit push --config=drizzle.config.ts",
     "drizzle:drop": "drizzle-kit drop --config=drizzle.config.ts",
     "test": "vitest run --mode=node",