@shapething/localstore 0.1.14

package/README.md

@@ -0,0 +1,27 @@
+# LocalStore
+
+A RDF/js Store that reads relative turtle files from disk and mutates them via a QueryEngine such as Comunica.
+
+## How to use:
+
+```TypeScript
+
+import { LocalStore } from '@shapething/localstore'
+import { QueryEngine } from '@comunica/query-sparql'
+
+const store = new LocalStore({ baseUri: new URL('http://example.com/') })
+const engine = new QueryEngine()
+await engine.queryQuads(
+    `construct { ?s ?p ?o } where {
+        { graph <http://example.com/nested/lorem> { ?s ?p ?o } } union
+        { graph <https://shapething.com/lorem> { ?s ?p ?o } } union
+        { graph <http://example.com/ipsum> { ?s ?p ?o } }
+    }`,
+    {
+      sources: [store]
+    }
+  )
+```
+
+It is best to execute queries with graphs selected.
+This gives the best performance.

package/dist/LocalStore.js

@@ -0,0 +1,286 @@
+import f from "@rdfjs/data-model";
+import m from "events";
+import v from "hex-encoding";
+import { get as y, set as E } from "idb-keyval";
+import { Store as w, Parser as S, Writer as N } from "n3";
+import { Readable as T } from "readable-stream";
+async function* x(a, t = ".", e = []) {
+  for await (const n of a.values())
+    if (n.kind === "file" && n.name.endsWith(".ttl"))
+      yield [`${t}/${n.name}`, n];
+    else if (n.kind === "directory") {
+      if (e.includes(n.name)) continue;
+      for await (const [r, s] of x(n, `${t}/${n.name}`, e))
+        yield [r, s];
+    }
+}
+function $(a) {
+  return a && a !== null && a !== void 0;
+}
+const P = async (a, t, e) => {
+  const n = a.split("/").filter($);
+  let r = n.shift(), s = t;
+  for (; s && r; ) {
+    try {
+      n.length > 0 ? s = await s.getDirectoryHandle(r, { create: e }) : s = await s.getFileHandle(r, { create: e });
+    } catch {
+      try {
+        s = await s.getFileHandle(r, { create: e });
+      } catch {
+      }
+    }
+    r = n.shift();
+  }
+  return s;
+}, G = async (a, t, e) => {
+  const n = await P(a, t, e);
+  if (n.kind === "file")
+    return n;
+}, b = (a) => f.quad(a.subject, a.predicate, a.object);
+class D {
+  prefixes = {};
+  #t;
+  #n;
+  #e = new w();
+  #r = /* @__PURE__ */ new Map();
+  constructor({ baseUri: t }) {
+    if (this.#n = t, !t.toString().endsWith("/")) throw new Error("BaseIRI must end on a trailing slash");
+  }
+  /**
+   * Connects to a previously mounted folder when given its name,
+   * or show a directory picker to connect to a folder.
+   */
+  async mount(t) {
+    try {
+      const e = await y("mounts") ?? /* @__PURE__ */ new Set();
+      if (t && e.has(t)) {
+        this.#t = await y(t);
+        return;
+      }
+      if (t && !e.has(t) && console.info(`Could not find connection to folder ${t}, please connect again.`), this.#t = await globalThis.showDirectoryPicker(), !this.#t) throw new Error("Could not save the folder");
+      await E(t ?? this.#t.name, this.#t), e.add(t ?? this.#t.name), E("mounts", e);
+    } catch (e) {
+      console.error(e);
+    }
+  }
+  /**
+   * Deletes a connection to a folder.
+   * This does not delete the folder on disk, but only the connection to it.
+   */
+  async unmount(t) {
+    try {
+      if (this.#t) {
+        this.#t = void 0;
+        const e = await y("mounts") ?? /* @__PURE__ */ new Set();
+        e.delete(t), E("mounts", e);
+      }
+    } catch (e) {
+      console.error(e);
+    }
+  }
+  /**
+   * Returns the name of the currently mounted folder.
+   */
+  getFolderName() {
+    return this.#t?.name;
+  }
+  /**
+   * Only use this when there is no data yet in the local folder attached to the previous baseURI.
+   * Currently this does not rename previous data, it orphans it.
+   */
+  setBaseUri(t) {
+    if (!t.toString().endsWith("/")) throw new Error("BaseURI must end on a trailing slash");
+    this.#n = t;
+  }
+  /**
+   * Removes quads by matching
+   */
+  removeMatches(t, e, n, r) {
+    const s = new m();
+    let i = 0, o = 0;
+    return (async () => {
+      const l = r ? [r] : this.getNamedGraphs();
+      for await (const c of l) {
+        i++, this.#e.removeMatches(t, e, n, c);
+        const h = this.match(t, e, n, c), d = [];
+        h.on("data", (u) => d.push(u)), h.on("end", () => {
+          this.updateGraph(c, { deletions: d }), o++, i === o && s.emit("end");
+        });
+      }
+    })(), s;
+  }
+  /**
+   * Deletes one graph, triggered by DROP GRAPH <a>
+   */
+  deleteGraph(t) {
+    const e = new m(), n = typeof t == "string" ? f.namedNode(t) : t;
+    return this.#e.deleteGraph(t), this.#s(n).then(async (r) => {
+      r?.remove(), e.emit("end");
+    }), e;
+  }
+  /**
+   * Imports quads to one or multiple disk files.
+   */
+  import(t) {
+    const e = new m();
+    let n;
+    const r = {}, s = (i) => {
+      n && (!i || !i.graph.equals(n)) && this.updateGraph(n, { insertions: r[n.value] }), i ? (i.graph.value in r || (r[i.graph.value] = []), r[i.graph.value].push(i), n = i.graph) : (n = void 0, e.emit("end"));
+    };
+    return t.on("data", s), t.on("end", s), t.on("error", (i) => {
+      console.error("Error in input stream:", i), e.emit("error", i);
+    }), e;
+  }
+  /**
+   * Removes quads from one or multiple disk files.
+   */
+  remove(t) {
+    const e = new m();
+    let n;
+    const r = {}, s = (i) => {
+      n && (!i || !i.graph.equals(n)) && this.updateGraph(n, { deletions: r[n.value] }), i ? (i.graph.value in r || (r[i.graph.value] = []), r[i.graph.value].push(i), n = i.graph) : (n = void 0, e.emit("end"));
+    };
+    return t.on("data", s), t.on("end", s), t.on("error", (i) => {
+      console.error("Error in input stream:", i), e.emit("error", i);
+    }), e;
+  }
+  /**
+   * Updates one graph on disk
+   */
+  async updateGraph(t, e) {
+    const n = await this.#s(t, !0), s = await (await n.getFile()).text(), o = new S({ baseIRI: t.value }).parse(s), l = new w(o), c = new w(e.deletions?.map(b)), h = new w(e.insertions?.map(b)), d = l.difference(c).union(h), u = new N({ baseIRI: t.value });
+    return u.addQuads([...d]), new Promise((p, g) => {
+      u.end(async (I, F) => {
+        const H = await n.createWritable();
+        await H.write(F), await H.close(), this.#e.deleteGraph(t), this.#r.delete(t.value), I && g(I), p(void 0);
+      });
+    });
+  }
+  /**
+   * An optimization for Comunica, so that there are less .match() calls.
+   */
+  countQuads(t, e, n, r) {
+    return r && this.#i(r) ? this.#e.countQuads(t, e, n, r) : 1;
+  }
+  /**
+   * The main function for local store for reading.
+   * When a match is done we decide which graphs we need to cache and parse them and put them in the N3 store.
+   */
+  match(t, e, n, r) {
+    if (!this.#t) throw new Error("Local store not mounted");
+    const s = new T({ objectMode: !0 });
+    let i = !1, o = !1, l = !1;
+    return s._read = async () => {
+      if (!(i || o)) {
+        i = !0;
+        try {
+          const c = this.getNamedGraphs(r ? [r] : void 0);
+          let h = 0, d = 0;
+          for await (const u of c) {
+            if (o) return;
+            h++, this.#i(u) || await this.#o(u);
+            const p = this.#e.match(
+              /** @ts-expect-error the N3 Store#match typings don't accept nullable term filters here */
+              t,
+              e,
+              n,
+              u
+            );
+            p.on("data", (g) => {
+              s.destroyed || o || s.push(g);
+            }), p.on("end", () => {
+              d++, h === d && !o && l && (o = !0, s.push(null));
+            });
+          }
+          l = !0, !o && l && h === d && (o = !0, s.push(null));
+        } catch (c) {
+          o || (o = !0, s.destroy(c));
+        }
+      }
+    }, s;
+  }
+  /**
+   * Check in the N3 store if the graph is cached.
+   */
+  #i(t) {
+    return this.#e._graphs[this.#e._termToNumericId(t)] !== void 0;
+  }
+  /**
+   * Cache a graph, it can be the case that a graph on disk is parsed and that this takes a while and in the same time a request is done for the same graph.
+   * For this reason we deduplicate the in flight promises.
+   */
+  async #o(t) {
+    const e = await this.#s(t);
+    if (e) {
+      if (!this.#r.has(t.value)) {
+        const n = this.#a(t, e);
+        this.#r.set(t.value, n);
+      }
+      return this.#r.get(t.value);
+    }
+  }
+  /**
+   * The actual function that parses a file on disk and puts it into the N3 store.
+   */
+  async #a(t, e) {
+    try {
+      const n = new S({ baseIRI: t.value }), s = (await (await e.getFile()).text()).replace(/<(.|\/)(.*)\.ttl(.*)>/g, "<$1$2>"), i = await n.parse(s), o = n._prefixes;
+      this.prefixes = { ...this.prefixes, ...o }, this.#e.addQuads(i.map((l) => f.quad(l.subject, l.predicate, l.object, t)));
+    } catch (n) {
+      throw new Error(`Error while parsing graph ${t.value}:` + (n instanceof Error ? n.message : String(n)));
+    }
+  }
+  /**
+   * Returns all named graphs or a subset of them.
+   */
+  async *getNamedGraphs(t) {
+    if (t && t.length > 0) {
+      for (const e of t)
+        await this.#s(e, !1) && (yield e);
+      return;
+    }
+    for await (const [e] of this.#l())
+      yield e;
+  }
+  async *#l() {
+    if (!this.#t) throw new Error("Local store not mounted");
+    for await (const [t, e] of x(this.#t))
+      yield [this.#c(t), e];
+  }
+  /**
+   * Given a path and a file handle, creates a graph term.
+   */
+  #c(t) {
+    const e = t.substring(0, t.length - 4), r = e.split("/").pop();
+    if (t === "default-graph.ttl") return f.defaultGraph();
+    if (v.is(r)) {
+      const s = v.decodeStr(r);
+      return f.namedNode(s);
+    }
+    return f.namedNode(new URL(e, this.#n).toString());
+  }
+  /**
+   * Given a graph term returns the file handle from disk.
+   */
+  async #s(t, e) {
+    if (!this.#t) throw new Error("Local store not mounted");
+    if (t.value.startsWith(this.#n.toString())) {
+      const n = t.value.replace(this.#n.toString(), ""), r = n.endsWith("/") ? n.substring(0, n.length - 1) : n, s = `${r}.ttl`;
+      if (!r) throw new Error("A graph must have a filename");
+      try {
+        return G(s, this.#t, e);
+      } catch {
+      }
+    } else {
+      if (t.termType === "DefaultGraph")
+        return G("default-graph.ttl", this.#t, e);
+      {
+        const n = new TextEncoder().encode(t.value), s = `${v.encode(n)}.ttl`;
+        return G(s, this.#t, e);
+      }
+    }
+  }
+}
+export {
+  D as LocalStore
+};

package/dist/lib/LocalStore.d.ts

@@ -0,0 +1,70 @@
+import type { DefaultGraph, NamedNode, Quad, Quad_Graph, Store as RdfJsStore, Source, Stream, Term } from '@rdfjs/types';
+import EventEmitter from 'events';
+type LocalStoreOptions = {
+    baseUri: URL;
+};
+/**
+ * Creates a queryable store, that mounts a directory on the drive and read and writes turtle files.
+ * Each of these files can have relative IRIs such as <> or <#lorem> or </ipsum>.
+ */
+export declare class LocalStore implements Source, RdfJsStore {
+    #private;
+    prefixes: Record<string, string>;
+    constructor({ baseUri }: LocalStoreOptions);
+    /**
+     * Connects to a previously mounted folder when given its name,
+     * or show a directory picker to connect to a folder.
+     */
+    mount(name?: string): Promise<void>;
+    /**
+     * Deletes a connection to a folder.
+     * This does not delete the folder on disk, but only the connection to it.
+     */
+    unmount(name: string): Promise<void>;
+    /**
+     * Returns the name of the currently mounted folder.
+     */
+    getFolderName(): string | undefined;
+    /**
+     * Only use this when there is no data yet in the local folder attached to the previous baseURI.
+     * Currently this does not rename previous data, it orphans it.
+     */
+    setBaseUri(baseUri: URL): void;
+    /**
+     * Removes quads by matching
+     */
+    removeMatches(subject?: Term | null, predicate?: Term | null, object?: Term | null, graph?: Term | null): EventEmitter;
+    /**
+     * Deletes one graph, triggered by DROP GRAPH <a>
+     */
+    deleteGraph(graph: string | Quad_Graph): EventEmitter;
+    /**
+     * Imports quads to one or multiple disk files.
+     */
+    import(stream: Stream<Quad>): EventEmitter;
+    /**
+     * Removes quads from one or multiple disk files.
+     */
+    remove(stream: Stream<Quad>): EventEmitter;
+    /**
+     * Updates one graph on disk
+     */
+    updateGraph(graph: NamedNode | DefaultGraph, update: {
+        deletions?: Quad[];
+        insertions?: Quad[];
+    }): Promise<void>;
+    /**
+     * An optimization for Comunica, so that there are less .match() calls.
+     */
+    countQuads(subject?: Term | null, predicate?: Term | null, object?: Term | null, graph?: Term | null): number;
+    /**
+     * The main function for local store for reading.
+     * When a match is done we decide which graphs we need to cache and parse them and put them in the N3 store.
+     */
+    match(subject?: Term | null, predicate?: Term | null, object?: Term | null, graph?: Term | null): Stream<Quad>;
+    /**
+     * Returns all named graphs or a subset of them.
+     */
+    getNamedGraphs(graphs?: (NamedNode | DefaultGraph)[]): AsyncIterable<NamedNode | DefaultGraph>;
+}
+export {};

package/dist/lib/helpers/getAllFilesFromDirectory.d.ts

@@ -0,0 +1 @@
+export declare function getAllFilesFromDirectory(handle: FileSystemDirectoryHandle, parentPath?: string, skipList?: string[]): AsyncIterable<[string, FileSystemFileHandle]>;

package/dist/lib/helpers/getFileHandleByPath.d.ts

@@ -0,0 +1,2 @@
+export declare const getFileHandleByPath: (path: string, root: FileSystemDirectoryHandle, create?: boolean) => Promise<FileSystemFileHandle | undefined>;
+export declare const getDirectoryHandleByPath: (path: string, root: FileSystemDirectoryHandle, create?: boolean) => Promise<FileSystemDirectoryHandle>;

package/dist/lib/helpers/nonNullable.d.ts

@@ -0,0 +1 @@
+export declare function nonNullable<T>(value: T): value is NonNullable<T>;

package/dist/lib/helpers/toTriple.d.ts

@@ -0,0 +1,2 @@
+import type { Quad } from '@rdfjs/types';
+export declare const toTriple: (quad: Quad) => Quad;

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@shapething/localstore",
+  "version": "0.1.14",
+  "type": "module",
+  "license": "MIT",
+  "main": "./dist/LocalStore.js",
+  "types": "./dist/lib/LocalStore.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/lib/LocalStore.d.ts",
+      "default": "./dist/LocalStore.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": ["dist", "README.md"],
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build --config vite.lib.config.ts && tsc -p tsconfig.build.json",
+    "lint": "eslint .",
+    "test": "vitest run --passWithNoTests",
+    "test:watch": "vitest"
+  },
+  "dependencies": {
+    "@rdfjs/data-model": "^2.1.0",
+    "events": "^3.3.0",
+    "hex-encoding": "^2.0.3",
+    "idb-keyval": "^6.2.2",
+    "n3": "^1.25.2",
+    "readable-stream": "^4.7.0"
+  },
+  "devDependencies": {
+    "@comunica/query-sparql": "^4.2.0",
+    "@rdfjs/types": "^2.0.1",
+    "@testing-library/dom": "^10.4.0",
+    "@types/events": "^3.0.3",
+    "@types/n3": "^1.25.3",
+    "@types/node": "^22.15.29",
+    "@types/rdfjs__data-model": "^2.0.9",
+    "@types/wicg-file-system-access": "^2023.10.6",
+    "@vitest/browser": "^3.1.4",
+    "typescript": "~5.8.3",
+    "vite": "^6.3.5",
+    "vitest": "^3.1.4"
+  }
+}