alinea 2.0.0-preview.11 → 2.0.0-preview.13

package/dist/.init/content/pages/welcome.json

@@ -1,5 +1,5 @@
 {
-  "_id": "3KHB2PydgBPbUsKveWcaGSXkqjV",
+  "_id": "3Kd9mGx2n7q2f3qhMUmFsgaYVyu",
   "_type": "Page",
   "_index": "a0",
   "_seeded": "welcome.json",

package/dist/.mcp/later/dev.json

@@ -1 +1 @@
-{"url":"http://localhost:1","pid":3065}
+{"url":"http://localhost:1","pid":3062}

package/dist/.mcp/online/dev.json

@@ -1 +1 @@
-{"url":"http://localhost:38409","pid":3065}
+{"url":"http://localhost:36863","pid":3062}

package/dist/adapter/next/cms.js

@@ -8,10 +8,10 @@ import {
   applyPreview as applyPreviewUpdate,
   decodePreviewRequest
 } from "alinea/backend/resolver/ParsePreview";
+import { dashboardUrl } from "./dashboardUrl.js";
 import { createThrottledSync } from "alinea/backend/util/Syncable";
 import { Client } from "alinea/core/Client";
 import { CMS } from "alinea/core/CMS";
-import { Config } from "alinea/core/Config";
 import {
   withDevelopmentKey
 } from "alinea/core/Connection";
@@ -24,7 +24,7 @@ import { cache } from "react";
 import { requestContext } from "./context.js";
 import { cachedResolve, developmentRevision } from "./devCache.js";
 import { RenderStats, summarizeQuery, timed } from "./renderStats.js";
-import { syncIfStale } from "./syncCheck.js";
+import { contentShaWindow, syncIfStale, tagContentSha } from "./syncCheck.js";
 import { jsx } from "react/jsx-runtime";
 var preValidatedRemote = { validate: false };
 var NextCMS = class extends CMS {
@@ -66,6 +66,10 @@ var NextCMS = class extends CMS {
    * returns a fresh holder, so nothing is recorded there.
    */
   #render = cache(() => ({}));
+  /** Build renders join the content sha tag once, see `tagContentSha`. */
+  #tagContent = cache(
+    () => tagContentSha(contentShaWindow(this.config.syncInterval))
+  );
   /** One content revision per render, see `developmentRevision`. */
   #devRevision = cache(() => developmentRevision());
   #isDraft = cache(async () => {
@@ -201,12 +205,19 @@ var NextCMS = class extends CMS {
     const answer = () => stats && row ? stats.executed(row, () => db.resolve(request)) : db.resolve(request);
     const syncInterval = request.disableSync ? Number.POSITIVE_INFINITY : request.syncInterval ?? this.config.syncInterval;
     if (hasPreview) return answer();
-    if (!isBuild) {
+    if (isBuild) {
+      await this.#tagContent();
+    } else {
       const tracked = {
         sha: db.sha,
         syncWith: () => this.#syncDb(db, client, stats)
       };
-      const settled = !isDraft && await syncIfStale(tracked, client, syncInterval);
+      const settled = !isDraft && await syncIfStale(
+        tracked,
+        client,
+        syncInterval,
+        contentShaWindow(this.config.syncInterval)
+      );
       if (!settled)
         await this.throttle(() => this.#syncDb(db, client, stats), syncInterval);
     }
@@ -264,16 +275,13 @@ var NextCMS = class extends CMS {
     if (!stats) return null;
     const { default: dynamic } = await import("next/dynamic.js");
     const { isDev, handlerUrl } = await requestContext(this.config);
-    let adminPath = Config.adminPath(this.config);
-    if (!adminPath.startsWith("/")) adminPath = `/${adminPath}`;
-    const dashboardUrl = isDev ? adminPath : new URL(`${adminPath}.html`, handlerUrl).href;
     const NextPreviews = dynamic(() => import("./previews.js"), {
       ssr: false
     });
     return /* @__PURE__ */ jsx(
       NextPreviews,
       {
-        dashboardUrl,
+        dashboardUrl: dashboardUrl(this.config, isDev, handlerUrl),
         widget,
         stats: widget && showStats ? stats.settled() : void 0,
         workspace,

package/dist/adapter/next/dashboardUrl.d.ts

@@ -0,0 +1,6 @@
+import { Config } from 'alinea/core/Config';
+/**
+ * The dashboard as the site serves it: in development the site proxies the
+ * admin path to the dev server, in production it serves the generated file.
+ */
+export declare function dashboardUrl(config: Config, isDev: boolean, handlerUrl: URL | string): string;

package/dist/adapter/next/dashboardUrl.js

@@ -0,0 +1,13 @@
+import "../../chunks/chunk-CQ52GEHY.js";
+
+// src/adapter/next/dashboardUrl.ts
+import { Config } from "alinea/core/Config";
+function dashboardUrl(config, isDev, handlerUrl) {
+  if (!isDev)
+    return new URL(`/${Config.dashboardFile(config)}`, handlerUrl).href;
+  const adminPath = Config.adminPath(config);
+  return adminPath.startsWith("/") ? adminPath : `/${adminPath}`;
+}
+export {
+  dashboardUrl
+};

package/dist/adapter/next/syncCheck.d.ts

@@ -6,13 +6,26 @@ interface SyncableDB {
     syncWith(source: RemoteSource, options?: SyncOptions): Promise<string>;
 }
 export declare const CONTENT_SHA_TAG = "alinea-content-sha";
+/**
+ * How long renders share the latest content sha. Configured through the
+ * config's `syncInterval`, never a query's: all renders share one entry.
+ */
+export declare function contentShaWindow(syncInterval?: number): number;
+/**
+ * Tag a static render with the content sha without looking it up. Renders at
+ * build skip `syncIfStale`, so this is how their pages join the tag that
+ * commits revalidate, and get the same time-based safety net as pages
+ * rendered at runtime.
+ */
+export declare function tagContentSha(revalidate?: number): Promise<void>;
 export declare function revalidateContentSha(): Promise<void>;
 /**
  * Sync the isolate-local db only when it is behind the shared content sha.
  * Returns true when the freshness question is settled here (synced, forced,
  * or explicitly disabled) and the caller can skip its throttled sync.
  * Returns false when the sha could not be determined, so the caller falls
- * back to its existing throttle behavior.
+ * back to its existing throttle behavior. `shaWindow` is how long renders
+ * share the latest sha, see `contentShaWindow`.
  */
-export declare function syncIfStale(db: SyncableDB, client: Client, syncInterval?: number, options?: SyncOptions): Promise<boolean>;
+export declare function syncIfStale(db: SyncableDB, client: Client, syncInterval?: number, shaWindow?: number, options?: SyncOptions): Promise<boolean>;
 export {};

package/dist/adapter/next/syncCheck.js

@@ -2,22 +2,27 @@ import "../../chunks/chunk-CQ52GEHY.js";
 
 // src/adapter/next/syncCheck.ts
 var CONTENT_SHA_TAG = "alinea-content-sha";
-var SHA_REVALIDATE_SECONDS = 60;
+var SHA_REVALIDATE_SECONDS = 300;
+function contentShaWindow(syncInterval) {
+  if (syncInterval !== void 0 && syncInterval > 0 && Number.isFinite(syncInterval))
+    return syncInterval;
+  return SHA_REVALIDATE_SECONDS;
+}
 var inflight;
-function latestSha(client) {
+function latestSha(client, revalidate) {
   if (inflight) return inflight;
-  inflight = loadLatestSha(client).finally(() => {
+  inflight = loadLatestSha(client, revalidate).finally(() => {
     inflight = void 0;
   });
   return inflight;
 }
-async function loadLatestSha(client) {
+async function loadLatestSha(client, revalidate) {
   try {
     const { unstable_cache } = await import("next/cache.js");
     const getSha = unstable_cache(
       () => client.getSha(),
       ["alinea-content-sha"],
-      { revalidate: SHA_REVALIDATE_SECONDS, tags: [CONTENT_SHA_TAG] }
+      { revalidate, tags: [CONTENT_SHA_TAG] }
     );
     return await getSha();
   } catch {
@@ -28,26 +33,38 @@ async function loadLatestSha(client) {
     }
   }
 }
+async function tagContentSha(revalidate = SHA_REVALIDATE_SECONDS) {
+  try {
+    const { unstable_cache } = await import("next/cache.js");
+    await unstable_cache(async () => true, ["alinea-content-sha-tag"], {
+      revalidate,
+      tags: [CONTENT_SHA_TAG]
+    })();
+  } catch {
+  }
+}
 async function revalidateContentSha() {
   try {
     const { revalidateTag } = await import("next/cache.js");
-    revalidateTag(CONTENT_SHA_TAG, { expire: 0 });
+    revalidateTag(CONTENT_SHA_TAG, "max");
   } catch {
   }
 }
-async function syncIfStale(db, client, syncInterval, options) {
+async function syncIfStale(db, client, syncInterval, shaWindow = SHA_REVALIDATE_SECONDS, options) {
   if (syncInterval === Number.POSITIVE_INFINITY) return true;
   if (syncInterval === 0) {
     await db.syncWith(client, options);
     return true;
   }
-  const expected = await latestSha(client);
+  const expected = await latestSha(client, shaWindow);
   if (!expected) return false;
   if (expected !== await db.sha) await db.syncWith(client, options);
   return true;
 }
 export {
   CONTENT_SHA_TAG,
+  contentShaWindow,
   revalidateContentSha,
-  syncIfStale
+  syncIfStale,
+  tagContentSha
 };

package/dist/backend/Handler.js

@@ -328,7 +328,11 @@ function createHandler({
       }
       if (action === HandleAction.Blob && request.method === "POST") {
         const { shas } = object({ shas: array(string) })(await body);
-        const tree = await local.source.getTree();
+        let tree = await local.source.getTree();
+        if (!dev && shas.some((sha) => !tree.hasSha(sha))) {
+          await syncForRead(cnx);
+          tree = await local.source.getTree();
+        }
         const fromLocal = [];
         const fromRemote = [];
         for (const sha of shas) {

package/dist/backend/api/BlobStore.d.ts

@@ -0,0 +1,21 @@
+import type { CommitApi, SyncApi } from 'alinea/core/Connection';
+/**
+ * Content blobs kept where every handler instance finds them, such as the
+ * backend database: an instance that is behind reads what another one
+ * fetched or committed, instead of fetching it from the remote again.
+ */
+export interface BlobStore {
+    /** The stored blobs among `shas`. */
+    readBlobs(shas: ReadonlyArray<string>): Promise<Map<string, Uint8Array>>;
+    /** Keep blobs, found again by their sha. */
+    storeBlobs(blobs: ReadonlyArray<[sha: string, blob: Uint8Array]>): Promise<void>;
+}
+export declare function isBlobStore<T extends object>(value: T): value is T & BlobStore;
+/**
+ * Answer from the store what it has, and fetch the rest from the remote,
+ * storing it. The store only saves requests: when it fails, the remote
+ * answers instead.
+ */
+export declare function storedBlobs(store: BlobStore, getBlobs: SyncApi['getBlobs']): SyncApi['getBlobs'];
+/** Commit through the remote, and store the committed content. */
+export declare function storedWrites(store: BlobStore, write: CommitApi['write']): CommitApi['write'];

package/dist/backend/api/BlobStore.js

@@ -0,0 +1,63 @@
+import "../../chunks/chunk-CQ52GEHY.js";
+
+// src/backend/api/BlobStore.ts
+import { hashBlob } from "alinea/core/source/GitUtils";
+var storeBatch = 100;
+function isBlobStore(value) {
+  return "readBlobs" in value && typeof value.readBlobs === "function" && "storeBlobs" in value && typeof value.storeBlobs === "function";
+}
+function storedBlobs(store, getBlobs) {
+  return async function* (shas, options) {
+    const stored = await store.readBlobs(shas).catch((error) => {
+      console.warn("Alinea could not read stored blobs", error);
+      return /* @__PURE__ */ new Map();
+    });
+    for (const sha of shas) {
+      const blob = stored.get(sha);
+      if (blob) yield [sha, blob];
+    }
+    const missing = shas.filter((sha) => !stored.has(sha));
+    if (missing.length === 0) return;
+    let fetched = Array();
+    const keep = async () => {
+      const blobs = fetched;
+      fetched = [];
+      if (blobs.length > 0) await keepBlobs(store, blobs);
+    };
+    try {
+      for await (const entry of getBlobs(missing, options)) {
+        fetched.push(entry);
+        if (fetched.length >= storeBatch) await keep();
+        yield entry;
+      }
+    } finally {
+      await keep();
+    }
+  };
+}
+function storedWrites(store, write) {
+  return async (request) => {
+    const result = await write(request);
+    const encoder = new TextEncoder();
+    const blobs = Array();
+    for (const change of request.changes) {
+      if (change.op !== "addContent") continue;
+      const blob = encoder.encode(change.contents);
+      if (await hashBlob(blob) === change.sha) blobs.push([change.sha, blob]);
+    }
+    if (blobs.length > 0) await keepBlobs(store, blobs);
+    return result;
+  };
+}
+async function keepBlobs(store, blobs) {
+  try {
+    await store.storeBlobs(blobs);
+  } catch (error) {
+    console.warn("Alinea could not store blobs", error);
+  }
+}
+export {
+  isBlobStore,
+  storedBlobs,
+  storedWrites
+};

package/dist/backend/api/CreateBackend.js

@@ -1,15 +1,15 @@
 import {
   driver_exports
-} from "../../chunks/chunk-QJGGYF6E.js";
-import "../../chunks/chunk-QSN6RH6X.js";
-import "../../chunks/chunk-ETWOY4AL.js";
-import "../../chunks/chunk-66YZ47TY.js";
-import "../../chunks/chunk-AIU6CMVT.js";
+} from "../../chunks/chunk-4ZA6KZHS.js";
+import "../../chunks/chunk-VA6NC3MI.js";
+import "../../chunks/chunk-A2I6MCCS.js";
+import "../../chunks/chunk-RAR44T5O.js";
+import "../../chunks/chunk-PTYZMV5G.js";
 import "../../chunks/chunk-N47Y62CI.js";
 import "../../chunks/chunk-QBPYWU6C.js";
 import "../../chunks/chunk-2BJ2IFQM.js";
 import "../../chunks/chunk-XCZXR3C2.js";
-import "../../chunks/chunk-MOWR65LM.js";
+import "../../chunks/chunk-PYDOMKJZ.js";
 import "../../chunks/chunk-76VMZTLN.js";
 import "../../chunks/chunk-2PIFHEYR.js";
 import "../../chunks/chunk-K62JP6HR.js";
@@ -20,6 +20,7 @@ import "../../chunks/chunk-CQ52GEHY.js";
 // src/backend/api/CreateBackend.ts
 import { assert } from "alinea/core/util/Assert";
 import { BasicAuth } from "./BasicAuth.js";
+import { isBlobStore, storedBlobs, storedWrites } from "./BlobStore.js";
 import { DatabaseApi } from "./DatabaseApi.js";
 import { GithubApi } from "./GithubApi.js";
 import { OAuth2 } from "./OAuth2.js";
@@ -65,13 +66,14 @@ function composeBackend(...impl) {
       throw new Error(`Backend does not implement ${name}`);
     };
   };
+  const blobStore = impl.find(isBlobStore);
   return {
     authenticate: call("authenticate"),
     verify: call("verify"),
     capabilities: call("capabilities"),
     getTreeIfDifferent: call("getTreeIfDifferent"),
-    getBlobs: call("getBlobs"),
-    write: call("write"),
+    getBlobs: blobStore ? storedBlobs(blobStore, call("getBlobs")) : call("getBlobs"),
+    write: blobStore ? storedWrites(blobStore, call("write")) : call("write"),
     revisions: call("revisions"),
     revisionData: call("revisionData"),
     getDraft: call("getDraft"),

package/dist/backend/api/DatabaseApi.d.ts

@@ -2,10 +2,11 @@ import type { DraftsApi, RequestContext, UploadResponse, UploadsApi, UserApi } f
 import { type Draft, type DraftKey } from 'alinea/core/Draft';
 import type { User, UserInput } from 'alinea/core/User';
 import { type Database } from 'rado';
+import type { BlobStore } from './BlobStore.js';
 export interface DatabaseOptions {
     db: Database;
 }
-export declare class DatabaseApi implements DraftsApi, UploadsApi, UserApi {
+export declare class DatabaseApi implements DraftsApi, UploadsApi, UserApi, BlobStore {
     #private;
     constructor(context: RequestContext, { db }: DatabaseOptions);
     getDraft(draftKey: DraftKey): Promise<Draft | undefined>;
@@ -13,6 +14,8 @@ export declare class DatabaseApi implements DraftsApi, UploadsApi, UserApi {
     prepareUpload(file: string): Promise<UploadResponse>;
     handleUpload(entryId: string, file: Blob): Promise<void>;
     previewUpload(entryId: string): Promise<Response>;
+    readBlobs(shas: ReadonlyArray<string>): Promise<Map<string, Uint8Array>>;
+    storeBlobs(blobs: ReadonlyArray<[sha: string, blob: Uint8Array]>): Promise<void>;
     enrichUser(user: User): Promise<User>;
     listUsers(): Promise<Array<User>>;
     createUser(user: UserInput): Promise<User>;

package/dist/backend/api/DatabaseApi.js

@@ -7,16 +7,21 @@ import {
   integer,
   text,
   varchar
-} from "../../chunks/chunk-ASYKC6IE.js";
+} from "../../chunks/chunk-5H4D5JQL.js";
 import {
   Builder
 } from "../../chunks/chunk-2BJ2IFQM.js";
 import {
-  eq
+  and,
+  eq,
+  inArray,
+  lt
 } from "../../chunks/chunk-XCZXR3C2.js";
 import {
+  Column,
+  ColumnType,
   table
-} from "../../chunks/chunk-MOWR65LM.js";
+} from "../../chunks/chunk-PYDOMKJZ.js";
 import "../../chunks/chunk-76VMZTLN.js";
 import "../../chunks/chunk-2PIFHEYR.js";
 import "../../chunks/chunk-K62JP6HR.js";
@@ -33,6 +38,7 @@ import "../../chunks/chunk-CQ52GEHY.js";
 import { createId } from "alinea/core/Id";
 import { assert } from "alinea/core/util/Assert";
 import { basename, extname } from "alinea/core/util/Paths";
+import { chunks } from "alinea/core/util/Arrays";
 import { slugify } from "alinea/core/util/Slugs";
 import { HandleAction } from "../HandleAction.js";
 var UploadTable = table("alinea_upload", {
@@ -48,7 +54,17 @@ var UserRoleTable = table("alinea_user_role", {
   userId: integer().notNull().references(UserTable.id),
   role: text().notNull()
 });
-var tables = [UploadTable, UserTable, UserRoleTable];
+var BlobTable = table("alinea_blob", {
+  sha: varchar(void 0, { length: 40 }).primaryKey(),
+  content: largeBlob().notNull(),
+  /** When the blob was last stored or read, in seconds */
+  usedAt: integer().notNull()
+});
+var tables = [UploadTable, UserTable, UserRoleTable, BlobTable];
+var blobsPerStatement = 50;
+var day = 24 * 60 * 60;
+var blobRetention = 30 * day;
+var blobCleanups = /* @__PURE__ */ new WeakMap();
 var preparedDatabases = /* @__PURE__ */ new WeakMap();
 var selectUser = {
   ...UserTable,
@@ -108,6 +124,35 @@ var DatabaseApi = class {
       }
     });
   }
+  async readBlobs(shas) {
+    const db = await this.#db;
+    const found = /* @__PURE__ */ new Map();
+    for (const batch of chunks(shas, blobsPerStatement)) {
+      const rows = await db.select({ sha: BlobTable.sha, content: BlobTable.content }).from(BlobTable).where(inArray(BlobTable.sha, batch));
+      for (const row of rows) found.set(row.sha, row.content);
+    }
+    const now = seconds();
+    for (const batch of chunks([...found.keys()], blobsPerStatement))
+      await renewBlobs(db, batch, now);
+    await removeUnusedBlobs(db, now);
+    return found;
+  }
+  async storeBlobs(blobs) {
+    const db = await this.#db;
+    const now = seconds();
+    const unique = [...new Map(blobs)];
+    for (const batch of chunks(unique, blobsPerStatement)) {
+      const shas = batch.map(([sha]) => sha);
+      const stored = new Set(
+        await db.select(BlobTable.sha).from(BlobTable).where(inArray(BlobTable.sha, shas))
+      );
+      await renewBlobs(db, [...stored], now);
+      const rows = batch.filter(([sha]) => !stored.has(sha)).map(([sha, content]) => ({ sha, content, usedAt: now }));
+      if (rows.length === 0) continue;
+      await db.insert(BlobTable).values(rows).catch(() => insertMissingBlobs(db, rows));
+    }
+    await removeUnusedBlobs(db, now);
+  }
   async enrichUser(user) {
     const normalized = normalizeUser(user);
     const found = await this.#getUser(normalized.email);
@@ -191,6 +236,45 @@ function prepareDatabase(db) {
   preparedDatabases.set(db, prepared);
   return prepared;
 }
+function largeBlob() {
+  return new Column({
+    type: new ColumnType(
+      "blob",
+      [],
+      sql.universal({
+        postgres: sql`bytea`,
+        mysql: sql`longblob`,
+        default: sql`blob`
+      })
+    ),
+    mapFromDriverValue(value) {
+      return new Uint8Array(value);
+    }
+  });
+}
+async function renewBlobs(db, shas, now) {
+  if (shas.length === 0) return;
+  await db.update(BlobTable).set({ usedAt: now }).where(and(inArray(BlobTable.sha, shas), lt(BlobTable.usedAt, now - day)));
+}
+async function insertMissingBlobs(db, rows) {
+  for (const row of rows) {
+    try {
+      await db.insert(BlobTable).values(row);
+    } catch (error) {
+      const stored = await db.select(BlobTable.sha).from(BlobTable).where(eq(BlobTable.sha, row.sha)).get();
+      if (!stored) throw error;
+    }
+  }
+}
+async function removeUnusedBlobs(db, now) {
+  const last = blobCleanups.get(db) ?? 0;
+  if (now - last < 60 * 60) return;
+  blobCleanups.set(db, now);
+  await db.delete(BlobTable).where(lt(BlobTable.usedAt, now - blobRetention));
+}
+function seconds() {
+  return Math.floor(Date.now() / 1e3);
+}
 async function enablePostgresRowLevelSecurity(db) {
   if (db.dialect.runtime !== "postgres") return;
   for (const table2 of tables) {