@nikala-ui/folio-algolia 0.1.0 → 0.2.0

package/README.md

@@ -10,7 +10,7 @@ Folio.
 
 ## Requirements
 
-- Folio `0.13.2` or newer;
+- Folio `0.14.0` or newer;
 - an Algolia index containing the same page URLs used by Folio;
 - an Algolia Search-Only API key.
 
@@ -66,8 +66,10 @@ are public by design; use only a Search-Only key.
 
 ## Index records
 
-Use the Folio page URL as Algolia's stable object ID. The helper preserves the
-page metadata needed by Folio:
+The indexing entrypoint is server-only. It accepts the page catalog produced by
+Folio and uploads records with an Algolia Admin API key. Use the Folio page URL
+as Algolia's stable object ID. The helper preserves the page metadata needed
+by Folio:
 
 ```ts
 import { toAlgoliaRecord } from "@nikala-ui/folio-algolia";
@@ -79,8 +81,68 @@ The generated record contains `objectID`, `url`, `slug`, `title`,
 `description`, `metadata` (frontmatter), and `headings` (table-of-contents
 text).
 
-Indexing is intentionally not part of this package. Run indexing from a
-separate trusted server or CI job with an Algolia Admin key.
+## Synchronize the index
+
+Create a script in the consuming Folio project, for example
+`scripts/index-search.ts`:
+
+```ts
+import path from "node:path";
+import { DEFAULT_DOCS_CONFIG, loadConfig } from "@nikala-ui/folio/config";
+import { scanContent } from "@nikala-ui/folio/content";
+import {
+  createAlgoliaIndexer,
+  getAlgoliaIndexerOptions,
+} from "@nikala-ui/folio-algolia/indexing";
+
+const projectRoot = process.cwd();
+const config = await loadConfig(projectRoot);
+const contentDir = path.resolve(
+  projectRoot,
+  config.contentDir ?? DEFAULT_DOCS_CONFIG.contentDir,
+);
+const pages = await scanContent(contentDir);
+const indexer = createAlgoliaIndexer(getAlgoliaIndexerOptions());
+const dryRun = process.argv.includes("--dry-run");
+const summary = await indexer.sync(pages, { dryRun });
+
+console.log(summary);
+```
+
+The indexing script runs outside the browser and reads these server-only
+variables:
+
+```bash
+ALGOLIA_APP_ID=your_application_id
+ALGOLIA_ADMIN_API_KEY=your_admin_key
+ALGOLIA_INDEX=your_index_name
+```
+
+Run a write-free validation first, then perform the upload:
+
+```bash
+bun run scripts/index-search.ts --dry-run
+bun run scripts/index-search.ts
+```
+
+The current sync operation uses Algolia's `updateObject` batch action, so
+re-running it safely updates existing records and creates missing records.
+
+Use full synchronization when records removed from the documentation should
+also be removed from Algolia:
+
+```ts
+const summary = await indexer.sync(pages, {
+  mode: "full",
+});
+```
+
+Full synchronization browses existing Algolia `objectID` values, compares
+them with the current Folio catalog, and deletes stale records. Transient
+`408`, `429`, and `5xx` responses are retried automatically. Configure
+`maxRetries` and `retryDelayMs` when the deployment environment needs a
+different policy. Use `continueOnError: true` only when the caller wants a
+summary containing failed batch counts instead of stopping at the first error.
 
 ## Manual adapter construction
 
@@ -97,16 +159,26 @@ const adapter = createAlgoliaAdapter({
 });
 ```
 
-For environment-based setup, the equivalent factory is available:
+Most Folio sites should use the exported `algoliaAdapter` instance instead.
+
+## Migration from 0.1.x
+
+Version `0.2.0` adds the server-only indexing entrypoint at
+`@nikala-ui/folio-algolia/indexing` and requires Folio `0.14.0` or newer for
+the configured content-directory workflow. Existing browser-side adapter
+configuration remains compatible:
 
 ```ts
-import { createAlgoliaAdapterFromEnv } from "@nikala-ui/folio-algolia";
+import { algoliaAdapter } from "@nikala-ui/folio-algolia";
 
-const adapter = createAlgoliaAdapterFromEnv();
+export default {
+  search: {
+    enabled: true,
+    provider: algoliaAdapter,
+  },
+};
 ```
 
-Most Folio sites should use the exported `algoliaAdapter` instance instead.
-
 ## Matching behavior
 
 Folio passes the query and its local page catalog to the adapter. Algolia
@@ -139,9 +211,9 @@ bun run build
 ```
 
 The package contains the Folio adapter contract, Algolia query client,
-environment resolution, and record mapping. Index creation, crawling, admin
-credentials, and deployment belong in the consuming project or a separate
-indexing service.
+server-side batch indexing, environment resolution, and record mapping.
+Crawling and deployment orchestration remain in the consuming project or a
+separate indexing service.
 
 ## License
 

package/dist/indexing/client.d.ts

@@ -0,0 +1,4 @@
+import type { AlgoliaIndexerOptions } from "./types.js";
+export declare function uploadBatch(options: AlgoliaIndexerOptions, records: Record<string, unknown>[]): Promise<void>;
+export declare function deleteBatch(options: AlgoliaIndexerOptions, objectIDs: string[]): Promise<void>;
+export declare function browseObjectIDs(options: AlgoliaIndexerOptions): Promise<string[]>;

package/dist/indexing/client.js

@@ -0,0 +1,76 @@
+import { AlgoliaIndexingError } from "./errors.js";
+const DEFAULT_INDEXING_ENDPOINT = "https://{appId}.algolia.net";
+const DEFAULT_MAX_RETRIES = 2;
+const DEFAULT_RETRY_DELAY_MS = 250;
+function createBatchUrl(endpoint, indexName) {
+    return `${endpoint.replace(/\/$/, "")}/1/indexes/${encodeURIComponent(indexName)}/batch`;
+}
+function resolveEndpoint(options) {
+    return options.endpoint
+        ?? DEFAULT_INDEXING_ENDPOINT.replace("{appId}", options.appId);
+}
+function isRetryableStatus(status) {
+    return status === 408 || status === 429 || status >= 500;
+}
+function wait(milliseconds) {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+function createHeaders(options) {
+    return {
+        "Content-Type": "application/json",
+        "X-Algolia-Application-Id": options.appId,
+        "X-Algolia-API-Key": options.adminApiKey,
+    };
+}
+async function requestWithRetry(options, url, init) {
+    const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
+    const retryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
+    const fetchImpl = options.fetch ?? globalThis.fetch;
+    for (let attempt = 0;; attempt += 1) {
+        const response = await fetchImpl(url, init);
+        if (response.ok || !isRetryableStatus(response.status) || attempt >= maxRetries) {
+            return response;
+        }
+        await wait(retryDelayMs * (attempt + 1));
+    }
+}
+async function sendBatch(options, requests) {
+    const response = await requestWithRetry(options, createBatchUrl(resolveEndpoint(options), options.indexName), {
+        method: "POST",
+        headers: createHeaders(options),
+        body: JSON.stringify({ requests }),
+    });
+    if (!response.ok) {
+        throw new AlgoliaIndexingError(`Algolia indexing failed with status ${response.status}`, response.status);
+    }
+}
+export async function uploadBatch(options, records) {
+    await sendBatch(options, records.map((body) => ({ action: "updateObject", body })));
+}
+export async function deleteBatch(options, objectIDs) {
+    await sendBatch(options, objectIDs.map((objectID) => ({
+        action: "deleteObject",
+        body: { objectID },
+    })));
+}
+export async function browseObjectIDs(options) {
+    const objectIDs = [];
+    let cursor;
+    do {
+        const response = await requestWithRetry(options, `${resolveEndpoint(options).replace(/\/$/, "")}/1/indexes/${encodeURIComponent(options.indexName)}/browse`, {
+            method: "POST",
+            headers: createHeaders(options),
+            body: JSON.stringify(cursor ? { cursor } : { attributesToRetrieve: ["objectID"] }),
+        });
+        if (!response.ok) {
+            throw new AlgoliaIndexingError(`Algolia browse failed with status ${response.status}`, response.status);
+        }
+        const data = await response.json();
+        for (const hit of data.hits ?? []) {
+            if (typeof hit.objectID === "string")
+                objectIDs.push(hit.objectID);
+        }
+        cursor = data.cursor;
+    } while (cursor);
+    return objectIDs;
+}

package/dist/indexing/env.d.ts

@@ -0,0 +1,4 @@
+import type { AlgoliaIndexerOptions } from "./types.js";
+type RuntimeEnv = Record<string, string | undefined>;
+export declare function getAlgoliaIndexerOptions(env?: RuntimeEnv): Pick<AlgoliaIndexerOptions, "appId" | "adminApiKey" | "indexName">;
+export {};

package/dist/indexing/env.js

@@ -0,0 +1,17 @@
+function getRuntimeEnv() {
+    const runtime = globalThis;
+    return runtime.process?.env ?? runtime.Bun?.env ?? {};
+}
+function requiredIndexerEnv(name, env) {
+    const value = env[name];
+    if (value)
+        return value;
+    throw new Error(`Missing required indexing environment variable: ${name}`);
+}
+export function getAlgoliaIndexerOptions(env = getRuntimeEnv()) {
+    return {
+        appId: requiredIndexerEnv("ALGOLIA_APP_ID", env),
+        adminApiKey: requiredIndexerEnv("ALGOLIA_ADMIN_API_KEY", env),
+        indexName: requiredIndexerEnv("ALGOLIA_INDEX", env),
+    };
+}

package/dist/indexing/errors.d.ts

@@ -0,0 +1,4 @@
+export declare class AlgoliaIndexingError extends Error {
+    readonly status?: number;
+    constructor(message: string, status?: number);
+}

package/dist/indexing/errors.js

@@ -0,0 +1,8 @@
+export class AlgoliaIndexingError extends Error {
+    status;
+    constructor(message, status) {
+        super(message);
+        this.name = "AlgoliaIndexingError";
+        this.status = status;
+    }
+}

package/dist/indexing/index.d.ts

@@ -0,0 +1,4 @@
+export { createAlgoliaIndexer } from "./sync.js";
+export { AlgoliaIndexingError } from "./errors.js";
+export { getAlgoliaIndexerOptions } from "./env.js";
+export type { AlgoliaIndexer, AlgoliaIndexerOptions, AlgoliaSyncOptions, AlgoliaSyncMode, AlgoliaSyncSummary, } from "./types.js";

package/dist/indexing/index.js

@@ -0,0 +1,3 @@
+export { createAlgoliaIndexer } from "./sync.js";
+export { AlgoliaIndexingError } from "./errors.js";
+export { getAlgoliaIndexerOptions } from "./env.js";

package/dist/indexing/sync.d.ts

@@ -0,0 +1,2 @@
+import type { AlgoliaIndexer, AlgoliaIndexerOptions } from "./types.js";
+export declare function createAlgoliaIndexer(options: AlgoliaIndexerOptions): AlgoliaIndexer;

package/dist/indexing/sync.js

@@ -0,0 +1,78 @@
+import { toAlgoliaRecord } from "../records.js";
+import { browseObjectIDs, deleteBatch, uploadBatch } from "./client.js";
+import { AlgoliaIndexingError } from "./errors.js";
+const DEFAULT_BATCH_SIZE = 1000;
+function assertServerRuntime() {
+    if (typeof window !== "undefined") {
+        throw new AlgoliaIndexingError("Algolia indexing is server-only and cannot run in a browser");
+    }
+}
+function splitIntoBatches(items, size) {
+    const batches = [];
+    for (let index = 0; index < items.length; index += size) {
+        batches.push(items.slice(index, index + size));
+    }
+    return batches;
+}
+function createSummary(total, batches, dryRun) {
+    return {
+        total,
+        batches,
+        uploaded: 0,
+        deleted: 0,
+        deleteBatches: 0,
+        failed: 0,
+        dryRun,
+    };
+}
+export function createAlgoliaIndexer(options) {
+    const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
+    if (!Number.isInteger(batchSize) || batchSize < 1) {
+        throw new TypeError("Algolia indexer batchSize must be a positive integer");
+    }
+    return {
+        async sync(pages, syncOptions = {}) {
+            assertServerRuntime();
+            const records = pages.map(toAlgoliaRecord);
+            const batches = splitIntoBatches(records, batchSize);
+            const mode = syncOptions.mode ?? "upsert";
+            const summary = createSummary(records.length, batches.length, Boolean(syncOptions.dryRun));
+            for (const batch of batches) {
+                if (syncOptions.dryRun)
+                    continue;
+                try {
+                    await uploadBatch(options, batch);
+                    summary.uploaded += batch.length;
+                }
+                catch (error) {
+                    summary.failed += batch.length;
+                    if (!syncOptions.continueOnError)
+                        throw error;
+                }
+            }
+            if (mode !== "full" || summary.failed > 0)
+                return summary;
+            const remoteObjectIDs = await browseObjectIDs(options);
+            const desiredObjectIDs = new Set(records.map((record) => String(record.objectID)));
+            const staleObjectIDs = remoteObjectIDs.filter((objectID) => !desiredObjectIDs.has(objectID));
+            const deleteBatches = splitIntoBatches(staleObjectIDs, batchSize);
+            summary.deleteBatches = deleteBatches.length;
+            for (const deleteBatchObjectIDs of deleteBatches) {
+                if (syncOptions.dryRun)
+                    continue;
+                try {
+                    await deleteBatch(options, deleteBatchObjectIDs);
+                    summary.deleted += deleteBatchObjectIDs.length;
+                }
+                catch (error) {
+                    summary.failed += deleteBatchObjectIDs.length;
+                    if (!syncOptions.continueOnError)
+                        throw error;
+                }
+            }
+            if (syncOptions.dryRun)
+                summary.deleted = staleObjectIDs.length;
+            return summary;
+        },
+    };
+}

package/dist/indexing/types.d.ts

@@ -0,0 +1,29 @@
+import type { FolioPage } from "../types.js";
+export interface AlgoliaIndexerOptions {
+    appId: string;
+    adminApiKey: string;
+    indexName: string;
+    endpoint?: string;
+    batchSize?: number;
+    maxRetries?: number;
+    retryDelayMs?: number;
+    fetch?: typeof globalThis.fetch;
+}
+export type AlgoliaSyncMode = "upsert" | "full";
+export interface AlgoliaSyncOptions {
+    dryRun?: boolean;
+    mode?: AlgoliaSyncMode;
+    continueOnError?: boolean;
+}
+export interface AlgoliaSyncSummary {
+    total: number;
+    batches: number;
+    uploaded: number;
+    deleted: number;
+    deleteBatches: number;
+    failed: number;
+    dryRun: boolean;
+}
+export interface AlgoliaIndexer {
+    sync(pages: FolioPage[], options?: AlgoliaSyncOptions): Promise<AlgoliaSyncSummary>;
+}

package/dist/indexing/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nikala-ui/folio-algolia",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Algolia search adapter for Folio documentation sites",
   "license": "MIT",
   "type": "module",
@@ -20,6 +20,10 @@
     ".": {
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
+    },
+    "./indexing": {
+      "types": "./dist/indexing/index.d.ts",
+      "import": "./dist/indexing/index.js"
     }
   },
   "files": ["dist"],
@@ -31,10 +35,10 @@
     "package:check": "bun run check && npm pack --dry-run"
   },
   "peerDependencies": {
-    "@nikala-ui/folio": ">=0.13.2"
+    "@nikala-ui/folio": ">=0.14.0"
   },
   "devDependencies": {
-    "@nikala-ui/folio": "^0.13.2",
+    "@nikala-ui/folio": "^0.14.0",
     "typescript": "^7.0.2"
   },
   "publishConfig": {