alinea 2.0.0-preview.12 → 2.0.0-preview.14

package/dist/backend/api/BlobStore.js

@@ -0,0 +1,63 @@
+import "../../chunks/chunk-CQ52GEHY.js";
+
+// src/backend/api/BlobStore.ts
+import { hashBlob } from "alinea/core/source/GitUtils";
+var storeBatch = 100;
+function isBlobStore(value) {
+  return "readBlobs" in value && typeof value.readBlobs === "function" && "storeBlobs" in value && typeof value.storeBlobs === "function";
+}
+function storedBlobs(store, getBlobs) {
+  return async function* (shas, options) {
+    const stored = await store.readBlobs(shas).catch((error) => {
+      console.warn("Alinea could not read stored blobs", error);
+      return /* @__PURE__ */ new Map();
+    });
+    for (const sha of shas) {
+      const blob = stored.get(sha);
+      if (blob) yield [sha, blob];
+    }
+    const missing = shas.filter((sha) => !stored.has(sha));
+    if (missing.length === 0) return;
+    let fetched = Array();
+    const keep = async () => {
+      const blobs = fetched;
+      fetched = [];
+      if (blobs.length > 0) await keepBlobs(store, blobs);
+    };
+    try {
+      for await (const entry of getBlobs(missing, options)) {
+        fetched.push(entry);
+        if (fetched.length >= storeBatch) await keep();
+        yield entry;
+      }
+    } finally {
+      await keep();
+    }
+  };
+}
+function storedWrites(store, write) {
+  return async (request) => {
+    const result = await write(request);
+    const encoder = new TextEncoder();
+    const blobs = Array();
+    for (const change of request.changes) {
+      if (change.op !== "addContent") continue;
+      const blob = encoder.encode(change.contents);
+      if (await hashBlob(blob) === change.sha) blobs.push([change.sha, blob]);
+    }
+    if (blobs.length > 0) await keepBlobs(store, blobs);
+    return result;
+  };
+}
+async function keepBlobs(store, blobs) {
+  try {
+    await store.storeBlobs(blobs);
+  } catch (error) {
+    console.warn("Alinea could not store blobs", error);
+  }
+}
+export {
+  isBlobStore,
+  storedBlobs,
+  storedWrites
+};

package/dist/backend/api/CreateBackend.js

@@ -1,15 +1,15 @@
 import {
   driver_exports
-} from "../../chunks/chunk-QJGGYF6E.js";
-import "../../chunks/chunk-QSN6RH6X.js";
-import "../../chunks/chunk-ETWOY4AL.js";
-import "../../chunks/chunk-66YZ47TY.js";
-import "../../chunks/chunk-AIU6CMVT.js";
+} from "../../chunks/chunk-6FQ5IMNA.js";
+import "../../chunks/chunk-A2I6MCCS.js";
+import "../../chunks/chunk-VA6NC3MI.js";
+import "../../chunks/chunk-RAR44T5O.js";
+import "../../chunks/chunk-PTYZMV5G.js";
 import "../../chunks/chunk-N47Y62CI.js";
 import "../../chunks/chunk-QBPYWU6C.js";
 import "../../chunks/chunk-2BJ2IFQM.js";
 import "../../chunks/chunk-XCZXR3C2.js";
-import "../../chunks/chunk-MOWR65LM.js";
+import "../../chunks/chunk-PYDOMKJZ.js";
 import "../../chunks/chunk-76VMZTLN.js";
 import "../../chunks/chunk-2PIFHEYR.js";
 import "../../chunks/chunk-K62JP6HR.js";
@@ -20,6 +20,7 @@ import "../../chunks/chunk-CQ52GEHY.js";
 // src/backend/api/CreateBackend.ts
 import { assert } from "alinea/core/util/Assert";
 import { BasicAuth } from "./BasicAuth.js";
+import { isBlobStore, storedBlobs, storedWrites } from "./BlobStore.js";
 import { DatabaseApi } from "./DatabaseApi.js";
 import { GithubApi } from "./GithubApi.js";
 import { OAuth2 } from "./OAuth2.js";
@@ -65,13 +66,14 @@ function composeBackend(...impl) {
       throw new Error(`Backend does not implement ${name}`);
     };
   };
+  const blobStore = impl.find(isBlobStore);
   return {
     authenticate: call("authenticate"),
     verify: call("verify"),
     capabilities: call("capabilities"),
     getTreeIfDifferent: call("getTreeIfDifferent"),
-    getBlobs: call("getBlobs"),
-    write: call("write"),
+    getBlobs: blobStore ? storedBlobs(blobStore, call("getBlobs")) : call("getBlobs"),
+    write: blobStore ? storedWrites(blobStore, call("write")) : call("write"),
     revisions: call("revisions"),
     revisionData: call("revisionData"),
     getDraft: call("getDraft"),

package/dist/backend/api/DatabaseApi.d.ts

@@ -2,10 +2,11 @@ import type { DraftsApi, RequestContext, UploadResponse, UploadsApi, UserApi } f
 import { type Draft, type DraftKey } from 'alinea/core/Draft';
 import type { User, UserInput } from 'alinea/core/User';
 import { type Database } from 'rado';
+import type { BlobStore } from './BlobStore.js';
 export interface DatabaseOptions {
     db: Database;
 }
-export declare class DatabaseApi implements DraftsApi, UploadsApi, UserApi {
+export declare class DatabaseApi implements DraftsApi, UploadsApi, UserApi, BlobStore {
     #private;
     constructor(context: RequestContext, { db }: DatabaseOptions);
     getDraft(draftKey: DraftKey): Promise<Draft | undefined>;
@@ -13,6 +14,8 @@ export declare class DatabaseApi implements DraftsApi, UploadsApi, UserApi {
     prepareUpload(file: string): Promise<UploadResponse>;
     handleUpload(entryId: string, file: Blob): Promise<void>;
     previewUpload(entryId: string): Promise<Response>;
+    readBlobs(shas: ReadonlyArray<string>): Promise<Map<string, Uint8Array>>;
+    storeBlobs(blobs: ReadonlyArray<[sha: string, blob: Uint8Array]>): Promise<void>;
     enrichUser(user: User): Promise<User>;
     listUsers(): Promise<Array<User>>;
     createUser(user: UserInput): Promise<User>;

package/dist/backend/api/DatabaseApi.js

@@ -2,21 +2,25 @@ import {
   include
 } from "../../chunks/chunk-XTWYBWP2.js";
 import {
-  blob,
   id,
   integer,
   text,
   varchar
-} from "../../chunks/chunk-ASYKC6IE.js";
+} from "../../chunks/chunk-2PP7WJUJ.js";
 import {
   Builder
 } from "../../chunks/chunk-2BJ2IFQM.js";
 import {
-  eq
+  and,
+  eq,
+  inArray,
+  lt
 } from "../../chunks/chunk-XCZXR3C2.js";
 import {
+  Column,
+  ColumnType,
   table
-} from "../../chunks/chunk-MOWR65LM.js";
+} from "../../chunks/chunk-PYDOMKJZ.js";
 import "../../chunks/chunk-76VMZTLN.js";
 import "../../chunks/chunk-2PIFHEYR.js";
 import "../../chunks/chunk-K62JP6HR.js";
@@ -33,22 +37,35 @@ import "../../chunks/chunk-CQ52GEHY.js";
 import { createId } from "alinea/core/Id";
 import { assert } from "alinea/core/util/Assert";
 import { basename, extname } from "alinea/core/util/Paths";
+import { chunks } from "alinea/core/util/Arrays";
 import { slugify } from "alinea/core/util/Slugs";
 import { HandleAction } from "../HandleAction.js";
 var UploadTable = table("alinea_upload", {
   entryId: varchar(void 0, { length: 27 }).primaryKey(),
-  content: blob().notNull()
+  content: largeBlob().notNull()
 });
 var UserTable = table("alinea_user", {
   id: id(),
-  email: text().notNull().unique(),
+  email: indexedText().notNull().unique(),
   name: text()
 });
 var UserRoleTable = table("alinea_user_role", {
   userId: integer().notNull().references(UserTable.id),
   role: text().notNull()
 });
-var tables = [UploadTable, UserTable, UserRoleTable];
+var BlobTable = table("alinea_blob", {
+  sha: varchar(void 0, { length: 40 }).primaryKey(),
+  content: largeBlob().notNull(),
+  /** When the blob was last stored or read, in seconds */
+  usedAt: integer().notNull()
+});
+var tables = [UploadTable, UserTable, UserRoleTable, BlobTable];
+var maxParams = 100;
+var blobsPerStatement = 50;
+var blobsPerInsert = Math.floor(maxParams / 3);
+var day = 24 * 60 * 60;
+var blobRetention = 30 * day;
+var blobCleanups = /* @__PURE__ */ new WeakMap();
 var preparedDatabases = /* @__PURE__ */ new WeakMap();
 var selectUser = {
   ...UserTable,
@@ -108,6 +125,35 @@ var DatabaseApi = class {
       }
     });
   }
+  async readBlobs(shas) {
+    const db = await this.#db;
+    const found = /* @__PURE__ */ new Map();
+    for (const batch of chunks(shas, blobsPerStatement)) {
+      const rows = await db.select({ sha: BlobTable.sha, content: BlobTable.content }).from(BlobTable).where(inArray(BlobTable.sha, batch));
+      for (const row of rows) found.set(row.sha, row.content);
+    }
+    const now = seconds();
+    for (const batch of chunks([...found.keys()], blobsPerStatement))
+      await renewBlobs(db, batch, now);
+    await removeUnusedBlobs(db, now);
+    return found;
+  }
+  async storeBlobs(blobs) {
+    const db = await this.#db;
+    const now = seconds();
+    const unique = [...new Map(blobs)];
+    for (const batch of chunks(unique, blobsPerInsert)) {
+      const shas = batch.map(([sha]) => sha);
+      const stored = new Set(
+        await db.select(BlobTable.sha).from(BlobTable).where(inArray(BlobTable.sha, shas))
+      );
+      await renewBlobs(db, [...stored], now);
+      const rows = batch.filter(([sha]) => !stored.has(sha)).map(([sha, content]) => ({ sha, content, usedAt: now }));
+      if (rows.length === 0) continue;
+      await db.insert(BlobTable).values(rows).catch(() => insertMissingBlobs(db, rows));
+    }
+    await removeUnusedBlobs(db, now);
+  }
   async enrichUser(user) {
     const normalized = normalizeUser(user);
     const found = await this.#getUser(normalized.email);
@@ -191,6 +237,57 @@ function prepareDatabase(db) {
   preparedDatabases.set(db, prepared);
   return prepared;
 }
+function largeBlob() {
+  return new Column({
+    type: new ColumnType(
+      "blob",
+      [],
+      sql.universal({
+        postgres: sql`bytea`,
+        mysql: sql`longblob`,
+        default: sql`blob`
+      })
+    ),
+    mapFromDriverValue(value) {
+      return new Uint8Array(value);
+    }
+  });
+}
+function indexedText() {
+  return new Column({
+    type: new ColumnType(
+      "text",
+      [],
+      sql.universal({
+        mysql: sql`varchar(255)`,
+        default: sql`text`
+      })
+    )
+  });
+}
+async function renewBlobs(db, shas, now) {
+  if (shas.length === 0) return;
+  await db.update(BlobTable).set({ usedAt: now }).where(and(inArray(BlobTable.sha, shas), lt(BlobTable.usedAt, now - day)));
+}
+async function insertMissingBlobs(db, rows) {
+  for (const row of rows) {
+    try {
+      await db.insert(BlobTable).values(row);
+    } catch (error) {
+      const stored = await db.select(BlobTable.sha).from(BlobTable).where(eq(BlobTable.sha, row.sha)).get();
+      if (!stored) throw error;
+    }
+  }
+}
+async function removeUnusedBlobs(db, now) {
+  const last = blobCleanups.get(db) ?? 0;
+  if (now - last < 60 * 60) return;
+  blobCleanups.set(db, now);
+  await db.delete(BlobTable).where(lt(BlobTable.usedAt, now - blobRetention));
+}
+function seconds() {
+  return Math.floor(Date.now() / 1e3);
+}
 async function enablePostgresRowLevelSecurity(db) {
   if (db.dialect.runtime !== "postgres") return;
   for (const table2 of tables) {

package/dist/backend/api/GithubApi.d.ts

@@ -3,6 +3,19 @@ import type { EntryRecord } from 'alinea/core/EntryRecord';
 import type { CommitRequest } from 'alinea/core/db/CommitRequest';
 import { GithubSource, type GithubSourceOptions } from 'alinea/core/source/GithubSource';
 export interface GithubOptions extends GithubSourceOptions {
+    /**
+     * Commit as this author instead of the account that owns the token. Such
+     * commits are written through the REST API, which GraphQL commits do not
+     * allow, so GitHub does not mark them as verified. A save usually spends
+     * three REST requests, plus one per uploaded file. Removing a file that is
+     * already gone costs one more tree request, and a branch that moved in
+     * the meantime repeats the save.
+     */
+    author?: GithubCommitAuthor;
+}
+export interface GithubCommitAuthor {
+    name: string;
+    email: string;
 }
 export declare class GithubApi extends GithubSource implements HistoryApi, CommitApi, SyncApi {
     #private;