@quarb/wasm 0.28.0 → 0.30.0

package/README.md

@@ -4,11 +4,12 @@
 compiled to WebAssembly, with TypeScript types. One path
 language over JSON, YAML, TOML, CSV/TSV, XML, HTML, and
 Markdown — including the text-level reading of HTML, Markdown,
-and plain text (sections, paragraphs, quotes, lists) — running
-entirely client-side: no server, no native
-dependency, the same engine that powers the
-[playground](https://demo.quarb.org) and the Quarb Scraper
-browser extension.
+and plain text (sections, paragraphs, quotes, lists) — and the
+web level, a whole website as one queryable tree, running
+entirely client-side: no server, no native dependency, the same
+engine that powers the [playground](https://demo.quarb.org), the
+[search box on quarb.org](https://quarb.org/search.html), and
+the Quarb Scraper browser extension.
 
 ## Install
 
@@ -50,6 +51,47 @@ const envelope = JSON.parse(run('csv', csv, '/row @| count', Date.now()));
 // {ok: true, lines: ["891"]} — or {ok: false, error: "..."}
 ```
 
+## A website as one tree
+
+Pack a static site's pages into a tarball and the engine reads
+it at the web level: every page under
+`/sites/<host>/pages/<path>`, the links between pages resolved
+into `->link` / `<-link`, tags and categories the pages declare
+as rows of their own, and each page readable at the text level
+(sections, paragraphs). That is a site search with no server:
+
+```sh
+tar czf site.tgz -C public .      # every .html under public/
+```
+
+```ts
+import { openSite } from '@quarb/wasm';
+
+const bytes = await fetch('/site.tgz').then((r) => r.arrayBuffer());
+const site = await openSite(bytes, { base: 'https://example.org/' });
+
+// Full-text search: the paragraphs that mention both words,
+// each as a link record — title, href to the block, the text.
+site.query('//paragraph[:: == (/arbor.*?query/i)] | link | json');
+
+// The site's own structure.
+site.query('//page::title');                         // every page
+site.query('//page[::in_degree = 0]::path');         // orphans
+site.query('/sites/example.org/pages/about.html | <-link::path');
+site.query('//page @| top(5; ::pagerank) | %(::path; ::pagerank)');
+```
+
+`base` is the site's root URL; without it the engine reads the
+base off a page's `<link rel="canonical">` and falls back to
+`https://localhost/`. A site small enough to hold in memory
+needs no tarball: pass an object of path to HTML
+(`openSite({ 'index.html': html, 'about.html': html2 })`). Load
+once and query many times; a Web Worker keeps a slow query off
+the page thread (the search box on quarb.org is
+[built this way](https://quarb.org/articles/the-site-that-ranks-itself.html)).
+Pass `model` for a Quarb model file over the site (aliases,
+derived nodes, edges).
+
 ## Sessions: `@quarb/wasm/quai`
 
 The second entry point is the **session engine** — the build
@@ -93,8 +135,8 @@ authoritative reference.
 
 ## Scope
 
-This package bundles the text-format adapters listed above.
-The full engine — 40+ adapters from SQLite and Postgres to
+This package bundles the text-format adapters listed above, the
+text level, and the web level over a tarball. The full engine — 40+ adapters from SQLite and Postgres to
 Kafka, S3, and cloud log services, plus the `qua` CLI and the
 `quai` interactive session — ships as
 [Rust crates](https://crates.io/crates/quarb) and a

package/index.d.ts

@@ -58,3 +58,52 @@ export function run(
 
 /** The engine version, e.g. `"0.12.0"`. */
 export function version(): string;
+
+/**
+ * A website loaded once and queried many times at the web level:
+ * every page under `/sites/<host>/pages/<path>`, the tags and
+ * categories the pages declare as rows of their own, every
+ * hyperlink resolved (`->link`, `<-link`, `::in_degree`,
+ * `::pagerank`), each page readable at the text level
+ * (`//section`, `//paragraph`, `| link`). Construct with
+ * {@link openSite}.
+ */
+export class Site {
+  /**
+   * Run a Quarb query over the site. Returns the result lines
+   * (node results as locators such as
+   * `/sites/example.org/pages/about.html!/section[2]`); throws on
+   * a parse or execution error. `opts.now` pins `now()`.
+   */
+  query(q: string, opts?: { now?: number }): string[];
+  /** The raw JSON envelope of a query; never throws. */
+  run(q: string, now?: number): string;
+  /** Every page as a locator, sorted. */
+  entries(): string[];
+  /** Every page's locator with its declared `<title>`. */
+  titles(): Record<string, string>;
+  /** Release the wasm memory the site holds. */
+  free(): void;
+}
+
+/**
+ * Open a site: a tarball of its pages (`.tar` or `.tar.gz`, as
+ * bytes), or an object of site-relative path to HTML for a site
+ * small enough to hold in memory.
+ *
+ * `opts.base` is the site's root URL (`https://example.org/`)
+ * that page paths and relative links join against; by default it
+ * is read off a page's canonical URL, and is `https://localhost/`
+ * when no page declares one. `opts.model` is a Quarb model file
+ * over the site (aliases, derived nodes, edges). `opts.wasm`
+ * forwards to {@link initQuarb} when the engine is not yet
+ * initialized.
+ */
+export function openSite(
+  source: Uint8Array | ArrayBuffer | Record<string, string>,
+  opts?: {
+    base?: string;
+    model?: string;
+    wasm?: BufferSource | WebAssembly.Module | Response | Promise<Response>;
+  }
+): Promise<Site>;

package/index.js

@@ -6,7 +6,7 @@
 // envelope. The raw wasm-bindgen surface stays exported for
 // callers that want to manage initialization themselves.
 
-import init, { run, version } from './quarb_wasm.js';
+import init, { run, version, Site as WasmSite } from './quarb_wasm.js';
 
 let ready;
 
@@ -15,10 +15,10 @@ export function initQuarb(input) {
   if (!ready) {
     if (input === undefined && typeof process !== 'undefined' && process.versions?.node) {
       ready = import('node:fs/promises').then(async ({ readFile }) =>
-        init(await readFile(new URL('./quarb_wasm_bg.wasm', import.meta.url)))
+        init({ module_or_path: await readFile(new URL('./quarb_wasm_bg.wasm', import.meta.url)) })
       );
     } else {
-      ready = init(input);
+      ready = init(input === undefined ? undefined : { module_or_path: input });
     }
   }
   return ready;
@@ -35,4 +35,56 @@ export async function query(format, input, q, opts = {}) {
   return r.lines;
 }
 
+/**
+ * A website loaded once and queried many times at the web level:
+ * every page under `/sites/<host>/pages/<path>`, the tags and
+ * categories the pages declare as rows of their own, every
+ * hyperlink resolved (`->link`, `<-link`), each page readable at
+ * the text level (`//section`, `//paragraph`). Construct with
+ * {@link openSite}.
+ */
+export class Site {
+  #inner;
+  constructor(inner) { this.#inner = inner; }
+
+  /** Run a query over the site; the result lines, or a throw. */
+  query(q, opts = {}) {
+    const r = JSON.parse(this.#inner.query(q, opts.now ?? Date.now()));
+    if (!r.ok) throw new Error(r.error);
+    return r.lines;
+  }
+
+  /** The raw JSON envelope of a query; never throws. */
+  run(q, now = Date.now()) { return this.#inner.query(q, now); }
+
+  /** Every page as a locator, sorted. */
+  entries() { return JSON.parse(this.#inner.entries()); }
+
+  /** Every page's locator with its declared `<title>`. */
+  titles() { return JSON.parse(this.#inner.titles()); }
+
+  /** Release the wasm memory the site holds. */
+  free() { this.#inner.free(); }
+}
+
+/**
+ * Open a site. `source` is a tarball of its pages (`.tar` or
+ * `.tar.gz`, as bytes) or a plain object of site-relative path to
+ * HTML for a site small enough to hold in memory. `opts.base` is
+ * the site's root URL that paths and relative links join against
+ * (default: read off a page's canonical URL, else
+ * `https://localhost/`); `opts.model` a Quarb model file over the
+ * site; `opts.wasm` forwards to {@link initQuarb}.
+ */
+export async function openSite(source, opts = {}) {
+  await initQuarb(opts.wasm);
+  const base = opts.base ?? '';
+  const model = opts.model ?? '';
+  const inner =
+    source instanceof Uint8Array || source instanceof ArrayBuffer
+      ? WasmSite.open(source instanceof Uint8Array ? source : new Uint8Array(source), base, model)
+      : WasmSite.from_pages(JSON.stringify(source), base, model);
+  return new Site(inner);
+}
+
 export { run, version };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@quarb/wasm",
-  "version": "0.28.0",
-  "description": "Quarb \u2014 the arboreal query engine \u2014 compiled to WebAssembly, with TypeScript types. One-shot queries over JSON, YAML, TOML, CSV, XML, HTML, and Markdown, plus the quai session engine (multi-source mounts, joins, &N history, kaiv, SQLite) \u2014 entirely client-side.",
+  "version": "0.30.0",
+  "description": "Quarb \u2014 the arboreal query engine \u2014 compiled to WebAssembly, with TypeScript types. One-shot queries over JSON, YAML, TOML, CSV, XML, HTML, and Markdown, a whole website as one queryable tree (site search from a tarball, links, PageRank), plus the quai session engine (multi-source mounts, joins, &N history, kaiv, SQLite) \u2014 entirely client-side.",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -60,6 +60,9 @@
     "markdown",
     "jq",
     "xpath",
-    "scraping"
+    "scraping",
+    "site-search",
+    "static-site",
+    "pagerank"
   ]
 }

package/quai.js

@@ -15,10 +15,10 @@ export function initQuai(input) {
   if (!ready) {
     if (input === undefined && typeof process !== 'undefined' && process.versions?.node) {
       ready = import('node:fs/promises').then(async ({ readFile }) =>
-        init(await readFile(new URL('./quai_wasm_bg.wasm', import.meta.url)))
+        init({ module_or_path: await readFile(new URL('./quai_wasm_bg.wasm', import.meta.url)) })
       );
     } else {
-      ready = init(input);
+      ready = init(input === undefined ? undefined : { module_or_path: input });
     }
   }
   return ready;

package/quarb_wasm.d.ts

@@ -1,6 +1,64 @@
 /* tslint:disable */
 /* eslint-disable */
 
+/**
+ * A site held in memory: a tar or tar.gz of its pages at the
+ * web level — `/sites/<host>/pages/<path>` for every page, the
+ * tags and categories the pages declare as rows of their own,
+ * every hyperlink resolved (`->link`, `<-link`), each page
+ * grafted at the text level on entry. Load once, query many
+ * times.
+ */
+export class Site {
+    free(): void;
+    [Symbol.dispose](): void;
+    /**
+     * The pages as locators, for a quick listing without a query.
+     */
+    entries(): string;
+    /**
+     * The same site from pages already in memory: `pages` is a
+     * JSON object of site-relative path to HTML
+     * (`{"index.html": "<!doctype html>…", "about/index.html": …}`)
+     * — no tarball needed for a site small enough to hold in a
+     * string. `base_url` and `model` as in [`open`](Self::open).
+     */
+    static from_pages(pages: string, base_url: string, model: string): Site;
+    /**
+     * Load a tarball (gzipped or not). Pages join against the
+     * base read off a page's canonical URL. `text_level` is kept
+     * for the caller's sake: the web level always reads its
+     * pages as the reader's model.
+     */
+    constructor(bytes: Uint8Array, text_level: boolean);
+    /**
+     * Open a tarball (gzipped or not) with every option spelled
+     * out: `base_url` is the site's root (`https://example.org/`)
+     * that every page path and relative link joins against —
+     * empty to read it off a page's canonical URL, falling back
+     * to `https://localhost/` when no page declares one; `model`
+     * is a model file over the site, empty for none.
+     */
+    static open(bytes: Uint8Array, base_url: string, model: string): Site;
+    /**
+     * Run `query` over the site; the same JSON envelope as
+     * [`run`]. Node results render as locators
+     * (`/sites/quarb.org/pages/guides/jq.html!/section[2]`); a
+     * model's derived nodes as `/<container>/<role>[n]`.
+     */
+    query(query: string, now_millis: number): string;
+    /**
+     * Every page's locator with its declared title, as a JSON
+     * object — read from the store, no page parsed.
+     */
+    titles(): string;
+    /**
+     * Like [`new`](Self::new), with a model file over the site:
+     * aliases (`alias <s/^tag://i>;`), derived nodes, edges.
+     */
+    static with_model(bytes: Uint8Array, _text_level: boolean, model: string): Site;
+}
+
 /**
  * Execute `query` against `input` parsed as `format`
  * (json | yaml | toml | csv | tsv | xml | html | markdown |
@@ -19,7 +77,15 @@ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembl
 
 export interface InitOutput {
     readonly memory: WebAssembly.Memory;
+    readonly __wbg_site_free: (a: number, b: number) => void;
     readonly run: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
+    readonly site_entries: (a: number) => [number, number];
+    readonly site_from_pages: (a: number, b: number, c: number, d: number, e: number, f: number) => [number, number, number];
+    readonly site_new: (a: number, b: number, c: number) => [number, number, number];
+    readonly site_open: (a: number, b: number, c: number, d: number, e: number, f: number) => [number, number, number];
+    readonly site_query: (a: number, b: number, c: number, d: number) => [number, number];
+    readonly site_titles: (a: number) => [number, number];
+    readonly site_with_model: (a: number, b: number, c: number, d: number, e: number) => [number, number, number];
     readonly version: () => [number, number];
     readonly __wbindgen_exn_store: (a: number) => void;
     readonly __externref_table_alloc: () => number;
@@ -27,6 +93,7 @@ export interface InitOutput {
     readonly __wbindgen_malloc: (a: number, b: number) => number;
     readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
     readonly __wbindgen_free: (a: number, b: number, c: number) => void;
+    readonly __externref_table_dealloc: (a: number) => void;
     readonly __wbindgen_start: () => void;
 }
 

package/quarb_wasm.js

@@ -1,5 +1,176 @@
 /* @ts-self-types="./quarb_wasm.d.ts" */
 
+/**
+ * A site held in memory: a tar or tar.gz of its pages at the
+ * web level — `/sites/<host>/pages/<path>` for every page, the
+ * tags and categories the pages declare as rows of their own,
+ * every hyperlink resolved (`->link`, `<-link`), each page
+ * grafted at the text level on entry. Load once, query many
+ * times.
+ */
+export class Site {
+    static __wrap(ptr) {
+        const obj = Object.create(Site.prototype);
+        obj.__wbg_ptr = ptr;
+        SiteFinalization.register(obj, obj.__wbg_ptr, obj);
+        return obj;
+    }
+    __destroy_into_raw() {
+        const ptr = this.__wbg_ptr;
+        this.__wbg_ptr = 0;
+        SiteFinalization.unregister(this);
+        return ptr;
+    }
+    free() {
+        const ptr = this.__destroy_into_raw();
+        wasm.__wbg_site_free(ptr, 0);
+    }
+    /**
+     * The pages as locators, for a quick listing without a query.
+     * @returns {string}
+     */
+    entries() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const ret = wasm.site_entries(this.__wbg_ptr);
+            deferred1_0 = ret[0];
+            deferred1_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
+        }
+    }
+    /**
+     * The same site from pages already in memory: `pages` is a
+     * JSON object of site-relative path to HTML
+     * (`{"index.html": "<!doctype html>…", "about/index.html": …}`)
+     * — no tarball needed for a site small enough to hold in a
+     * string. `base_url` and `model` as in [`open`](Self::open).
+     * @param {string} pages
+     * @param {string} base_url
+     * @param {string} model
+     * @returns {Site}
+     */
+    static from_pages(pages, base_url, model) {
+        const ptr0 = passStringToWasm0(pages, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ptr1 = passStringToWasm0(base_url, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len1 = WASM_VECTOR_LEN;
+        const ptr2 = passStringToWasm0(model, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len2 = WASM_VECTOR_LEN;
+        const ret = wasm.site_from_pages(ptr0, len0, ptr1, len1, ptr2, len2);
+        if (ret[2]) {
+            throw takeFromExternrefTable0(ret[1]);
+        }
+        return Site.__wrap(ret[0]);
+    }
+    /**
+     * Load a tarball (gzipped or not). Pages join against the
+     * base read off a page's canonical URL. `text_level` is kept
+     * for the caller's sake: the web level always reads its
+     * pages as the reader's model.
+     * @param {Uint8Array} bytes
+     * @param {boolean} text_level
+     */
+    constructor(bytes, text_level) {
+        const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_malloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ret = wasm.site_new(ptr0, len0, text_level);
+        if (ret[2]) {
+            throw takeFromExternrefTable0(ret[1]);
+        }
+        this.__wbg_ptr = ret[0];
+        SiteFinalization.register(this, this.__wbg_ptr, this);
+        return this;
+    }
+    /**
+     * Open a tarball (gzipped or not) with every option spelled
+     * out: `base_url` is the site's root (`https://example.org/`)
+     * that every page path and relative link joins against —
+     * empty to read it off a page's canonical URL, falling back
+     * to `https://localhost/` when no page declares one; `model`
+     * is a model file over the site, empty for none.
+     * @param {Uint8Array} bytes
+     * @param {string} base_url
+     * @param {string} model
+     * @returns {Site}
+     */
+    static open(bytes, base_url, model) {
+        const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_malloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ptr1 = passStringToWasm0(base_url, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len1 = WASM_VECTOR_LEN;
+        const ptr2 = passStringToWasm0(model, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len2 = WASM_VECTOR_LEN;
+        const ret = wasm.site_open(ptr0, len0, ptr1, len1, ptr2, len2);
+        if (ret[2]) {
+            throw takeFromExternrefTable0(ret[1]);
+        }
+        return Site.__wrap(ret[0]);
+    }
+    /**
+     * Run `query` over the site; the same JSON envelope as
+     * [`run`]. Node results render as locators
+     * (`/sites/quarb.org/pages/guides/jq.html!/section[2]`); a
+     * model's derived nodes as `/<container>/<role>[n]`.
+     * @param {string} query
+     * @param {number} now_millis
+     * @returns {string}
+     */
+    query(query, now_millis) {
+        let deferred2_0;
+        let deferred2_1;
+        try {
+            const ptr0 = passStringToWasm0(query, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+            const len0 = WASM_VECTOR_LEN;
+            const ret = wasm.site_query(this.__wbg_ptr, ptr0, len0, now_millis);
+            deferred2_0 = ret[0];
+            deferred2_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+        }
+    }
+    /**
+     * Every page's locator with its declared title, as a JSON
+     * object — read from the store, no page parsed.
+     * @returns {string}
+     */
+    titles() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const ret = wasm.site_titles(this.__wbg_ptr);
+            deferred1_0 = ret[0];
+            deferred1_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
+        }
+    }
+    /**
+     * Like [`new`](Self::new), with a model file over the site:
+     * aliases (`alias <s/^tag://i>;`), derived nodes, edges.
+     * @param {Uint8Array} bytes
+     * @param {boolean} _text_level
+     * @param {string} model
+     * @returns {Site}
+     */
+    static with_model(bytes, _text_level, model) {
+        const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_malloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ptr1 = passStringToWasm0(model, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len1 = WASM_VECTOR_LEN;
+        const ret = wasm.site_with_model(ptr0, len0, _text_level, ptr1, len1);
+        if (ret[2]) {
+            throw takeFromExternrefTable0(ret[1]);
+        }
+        return Site.__wrap(ret[0]);
+    }
+}
+if (Symbol.dispose) Site.prototype[Symbol.dispose] = Site.prototype.free;
+
 /**
  * Execute `query` against `input` parsed as `format`
  * (json | yaml | toml | csv | tsv | xml | html | markdown |
@@ -50,6 +221,13 @@ export function version() {
 function __wbg_get_imports() {
     const import0 = {
         __proto__: null,
+        __wbg_Error_92b29b0548f8b746: function(arg0, arg1) {
+            const ret = Error(getStringFromWasm0(arg0, arg1));
+            return ret;
+        },
+        __wbg___wbindgen_throw_344f42d3211c4765: function(arg0, arg1) {
+            throw new Error(getStringFromWasm0(arg0, arg1));
+        },
         __wbg_getRandomValues_3f44b700395062e5: function() { return handleError(function (arg0, arg1) {
             globalThis.crypto.getRandomValues(getArrayU8FromWasm0(arg0, arg1));
         }, arguments); },
@@ -69,6 +247,10 @@ function __wbg_get_imports() {
     };
 }
 
+const SiteFinalization = (typeof FinalizationRegistry === 'undefined')
+    ? { register: () => {}, unregister: () => {} }
+    : new FinalizationRegistry(ptr => wasm.__wbg_site_free(ptr, 1));
+
 function addToExternrefTable0(obj) {
     const idx = wasm.__externref_table_alloc();
     wasm.__wbindgen_externrefs.set(idx, obj);
@@ -101,6 +283,13 @@ function handleError(f, args) {
     }
 }
 
+function passArray8ToWasm0(arg, malloc) {
+    const ptr = malloc(arg.length * 1, 1) >>> 0;
+    getUint8ArrayMemory0().set(arg, ptr / 1);
+    WASM_VECTOR_LEN = arg.length;
+    return ptr;
+}
+
 function passStringToWasm0(arg, malloc, realloc) {
     if (realloc === undefined) {
         const buf = cachedTextEncoder.encode(arg);
@@ -138,6 +327,12 @@ function passStringToWasm0(arg, malloc, realloc) {
     return ptr;
 }
 
+function takeFromExternrefTable0(idx) {
+    const value = wasm.__wbindgen_externrefs.get(idx);
+    wasm.__externref_table_dealloc(idx);
+    return value;
+}
+
 let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
 cachedTextDecoder.decode();
 const MAX_SAFARI_DECODE_BYTES = 2146435072;