@hatk/hatk 0.0.1-alpha.0

package/dist/feeds.d.ts

@@ -0,0 +1,95 @@
+import type { HydrateContext, Row } from './hydrate.ts';
+import type { Checked } from './lex-types.ts';
+export type { HydrateContext, Row };
+export interface FeedResult {
+    uris: string[];
+    cursor?: string;
+}
+export interface PaginateOpts {
+    params?: (string | number)[];
+    orderBy?: string;
+    order?: 'ASC' | 'DESC';
+}
+export interface PaginateResult<T> {
+    rows: T[];
+    cursor: string | undefined;
+}
+export interface FeedContext {
+    db: {
+        query: (sql: string, params?: any[]) => Promise<any[]>;
+    };
+    params: Record<string, string>;
+    cursor?: string;
+    limit: number;
+    viewer: {
+        did: string;
+    } | null;
+    packCursor: (primary: string | number, cid: string) => string;
+    unpackCursor: (cursor: string) => {
+        primary: string;
+        cid: string;
+    } | null;
+    isTakendown: (did: string) => Promise<boolean>;
+    filterTakendownDids: (dids: string[]) => Promise<Set<string>>;
+    paginate: <T extends {
+        uri: string;
+    }>(sql: string, opts?: PaginateOpts) => Promise<PaginateResult<T>>;
+}
+type FeedGenerate = (ctx: FeedContext & {
+    ok: (value: FeedResult) => Checked<FeedResult>;
+}) => Promise<Checked<FeedResult>>;
+type FeedOpts = {
+    collection: string;
+    view?: string;
+    label: string;
+    generate: FeedGenerate;
+    hydrate?: (ctx: HydrateContext<any>) => Promise<unknown[]>;
+} | {
+    collection?: never;
+    view?: never;
+    label: string;
+    generate: FeedGenerate;
+    hydrate: (ctx: HydrateContext<any>) => Promise<unknown[]>;
+};
+export declare function createPaginate(deps: {
+    db: {
+        query: (sql: string, params?: any[]) => Promise<any[]>;
+    };
+    cursor?: string;
+    limit: number;
+    packCursor: (primary: string | number, cid: string) => string;
+    unpackCursor: (cursor: string) => {
+        primary: string;
+        cid: string;
+    } | null;
+}): <T extends {
+    uri: string;
+}>(sql: string, opts?: PaginateOpts) => Promise<PaginateResult<T>>;
+export declare function defineFeed(opts: FeedOpts): {
+    generate: (ctx: any) => Promise<Checked<FeedResult>>;
+    collection: string;
+    view?: string;
+    label: string;
+    hydrate?: (ctx: HydrateContext<any>) => Promise<unknown[]>;
+} | {
+    generate: (ctx: any) => Promise<Checked<FeedResult>>;
+    collection?: never;
+    view?: never;
+    label: string;
+    hydrate: (ctx: HydrateContext<any>) => Promise<unknown[]>;
+};
+export declare function initFeeds(feedsDir: string): Promise<void>;
+/** Execute a feed and run its hydrate pipeline if present. */
+export declare function executeFeed(name: string, params: Record<string, string>, cursor: string | undefined, limit: number, viewer?: {
+    did: string;
+} | null): Promise<{
+    items?: unknown[];
+    uris?: string[];
+    cursor?: string;
+} | null>;
+export declare function listFeeds(): {
+    name: string;
+    label: string;
+}[];
+export declare function feedHasHydrate(name: string): boolean;
+//# sourceMappingURL=feeds.d.ts.map

package/dist/feeds.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"feeds.d.ts","sourceRoot":"","sources":["../src/feeds.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AAE7C,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,CAAA;AAEnC,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAA;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,EAAE,CAAA;IACT,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAC9D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC9B,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IACzE,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC9C,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7D,QAAQ,EAAE,CAAC,CAAC,SAAS;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,YAAY,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;CACtG;AAkBD,KAAK,YAAY,GAAG,CAClB,GAAG,EAAE,WAAW,GAAG;IAAE,EAAE,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,UAAU,CAAC,CAAA;CAAE,KAClE,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;AAEjC,KAAK,QAAQ,GACT;IACE,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CAC3D,GACD;IACE,UAAU,CAAC,EAAE,KAAK,CAAA;IAClB,IAAI,CAAC,EAAE,KAAK,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,EAAE,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CAC1D,CAAA;AAEL,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;CAC1E,IACe,CAAC,SAAS;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,EAAE,KAAK,MAAM,EAAE,OAAO,YAAY,KAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAmDvG;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,QAAQ;oBACL,GAAG;gBA3ErB,MAAM;WACX,MAAM;WACN,MAAM;cAEH,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;;oBAuE5B,GAAG;iBApEpB,KAAK;WACX,KAAK;WACL,MAAM;aAEJ,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;EAiE9D;AAID,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8D/D;AAED,8DAA8D;AAC9D,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC9B,OAAO,CAAC;IAAE,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAczE;AAED,wBAAgB,SAAS,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,CAE7D;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEpD"}

package/dist/feeds.js

@@ -0,0 +1,144 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { resolve } from 'node:path';
+import { readdirSync } from 'node:fs';
+import { log } from "./logger.js";
+import { querySQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./db.js";
+import { resolveRecords, buildHydrateContext } from "./hydrate.js";
+export function createPaginate(deps) {
+    return async (sql, opts) => {
+        const { db, cursor, limit, packCursor: pack, unpackCursor: unpack } = deps;
+        const userParams = opts?.params ?? [];
+        const orderBy = opts?.orderBy ?? 'indexed_at';
+        const order = opts?.order ?? 'DESC';
+        // Derive table prefix and bare column names from qualified orderBy (e.g. "p.played_time" → "p.")
+        const dotIdx = orderBy.lastIndexOf('.');
+        const prefix = dotIdx >= 0 ? orderBy.substring(0, dotIdx + 1) : '';
+        const cidCol = prefix + 'cid';
+        const orderByCol = dotIdx >= 0 ? orderBy.substring(dotIdx + 1) : orderBy;
+        const op = order === 'ASC' ? '>' : '<';
+        let paramIdx = userParams.length + 1;
+        const sqlParams = [...userParams];
+        // Build cursor condition
+        let cursorCondition = '';
+        if (cursor) {
+            const parsed = unpack(cursor);
+            if (parsed) {
+                cursorCondition = `(${orderBy} ${op} $${paramIdx} OR (${orderBy} = $${paramIdx + 1} AND ${cidCol} ${op} $${paramIdx + 2}))`;
+                sqlParams.push(parsed.primary, parsed.primary, parsed.cid);
+                paramIdx += 3;
+            }
+        }
+        // Detect existing WHERE and append cursor condition
+        let fullSql = sql;
+        if (cursorCondition) {
+            if (/\bWHERE\b/i.test(sql)) {
+                fullSql += ` AND ${cursorCondition}`;
+            }
+            else {
+                fullSql += ` WHERE ${cursorCondition}`;
+            }
+        }
+        fullSql += ` ORDER BY ${orderBy} ${order}, ${cidCol} ${order} LIMIT $${paramIdx}`;
+        sqlParams.push(limit + 1);
+        const rows = (await db.query(fullSql, sqlParams));
+        const hasMore = rows.length > limit;
+        if (hasMore)
+            rows.pop();
+        const last = rows[rows.length - 1];
+        return {
+            rows,
+            cursor: hasMore && last ? pack(last[orderByCol], last.cid) : undefined,
+        };
+    };
+}
+export function defineFeed(opts) {
+    return { ...opts, generate: (ctx) => opts.generate({ ...ctx, ok: (v) => v }) };
+}
+const feeds = new Map();
+export async function initFeeds(feedsDir) {
+    let files;
+    try {
+        files = readdirSync(feedsDir)
+            .filter((f) => (f.endsWith('.ts') || f.endsWith('.js')) && !f.startsWith('_'))
+            .sort();
+    }
+    catch {
+        return;
+    }
+    for (const file of files) {
+        const name = file.replace(/\.(ts|js)$/, '');
+        const scriptPath = resolve(feedsDir, file);
+        const mod = await import(__rewriteRelativeImportExtension(scriptPath));
+        const generator = mod.default;
+        const handler = {
+            name,
+            label: generator.label || name,
+            collection: generator.collection,
+            view: generator.view,
+            generate: async (params, cursor, limit, viewer) => {
+                const paginateDeps = {
+                    db: { query: querySQL },
+                    cursor,
+                    limit,
+                    packCursor,
+                    unpackCursor,
+                };
+                const ctx = {
+                    db: { query: querySQL },
+                    params,
+                    cursor,
+                    limit,
+                    viewer,
+                    packCursor,
+                    unpackCursor,
+                    isTakendown: isTakendownDid,
+                    filterTakendownDids,
+                    paginate: createPaginate(paginateDeps),
+                };
+                const result = await generator.generate(ctx);
+                if (Array.isArray(result)) {
+                    return { uris: result.map((r) => r.uri || r) };
+                }
+                return {
+                    uris: result.uris,
+                    cursor: result.cursor,
+                };
+            },
+        };
+        if (typeof generator.hydrate === 'function') {
+            handler.hydrate = generator.hydrate;
+            log(`[feeds] discovered: ${name} (with hydrate)`);
+        }
+        else {
+            log(`[feeds] discovered: ${name}`);
+        }
+        feeds.set(name, handler);
+    }
+}
+/** Execute a feed and run its hydrate pipeline if present. */
+export async function executeFeed(name, params, cursor, limit, viewer) {
+    const handler = feeds.get(name);
+    if (!handler)
+        return null;
+    const result = await handler.generate(params, cursor, limit, viewer || null);
+    if (handler.hydrate) {
+        const items = await resolveRecords(result.uris);
+        const ctx = buildHydrateContext(items, viewer || null);
+        const hydrated = await handler.hydrate(ctx);
+        return { items: hydrated, cursor: result.cursor };
+    }
+    return { uris: result.uris, cursor: result.cursor };
+}
+export function listFeeds() {
+    return Array.from(feeds.values()).map((f) => ({ name: f.name, label: f.label }));
+}
+export function feedHasHydrate(name) {
+    return feeds.get(name)?.hydrate != null;
+}

package/dist/fts.d.ts

@@ -0,0 +1,20 @@
+export declare function getSearchColumns(collection: string): string[];
+export declare function getLastRebuiltAt(collection: string): string | null;
+/**
+ * DuckDB FTS can't handle dots in table names (interprets them as catalog.schema.table).
+ * We create a shadow table with underscored names for FTS indexing.
+ */
+export declare function ftsTableName(collection: string): string;
+/**
+ * Build FTS index for a collection.
+ * Creates a shadow table copy and indexes all TEXT NOT NULL columns
+ * using Porter stemmer with English stopwords.
+ */
+export declare function buildFtsIndex(collection: string): Promise<void>;
+/**
+ * Strip English stop words from a search query, preserving non-stop-word terms.
+ * Returns the cleaned query string. If all words are stop words, returns the original query.
+ */
+export declare function stripStopWords(query: string): string;
+export declare function rebuildAllIndexes(collections: string[]): Promise<void>;
+//# sourceMappingURL=fts.d.ts.map

package/dist/fts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB5E"}