@thomasfosterau/effect-rdf 0.0.0-stage → 0.1.0

package/src/sparql/GraphStore.ts

@@ -0,0 +1,341 @@
+/**
+ * The SPARQL 1.1 Graph Store HTTP Protocol: `GET`, `HEAD`, `PUT`, `POST` and
+ * `DELETE` on graphs identified indirectly (`?default`, `?graph=<iri>`) or
+ * directly (the request URL is the graph IRI), and `POST` to the graph store
+ * itself to create a new graph (`201 Created` with its IRI in `Location`).
+ *
+ * Request bodies are parsed with the `Serialization.Formats` parser for their
+ * `Content-Type` (`multipart/form-data` bodies are parsed part by part);
+ * responses are serialised with the negotiated serializer.
+ *
+ * @since 0.1.0
+ */
+import * as Match from 'effect/Match';
+import * as Arr from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Predicate from 'effect/Predicate';
+import * as Option from 'effect/Option';
+import * as Stream from 'effect/Stream';
+import * as HttpServerRequest from 'effect/http/HttpServerRequest';
+import * as HttpServerResponse from 'effect/http/HttpServerResponse';
+import * as Iri from '../Iri.ts';
+import * as PrefixMap from '../PrefixMap.ts';
+import * as Serialization from '../Serialization.ts';
+import * as Term from '../Term.ts';
+import * as Store from './Store.ts';
+import * as ProtocolErrors from './ProtocolErrors.ts';
+import * as ServerFormats from './ServerFormats.ts';
+import * as Options from './Options.ts';
+import * as Protocol from './Protocol.ts';
+
+/**
+ * The graph a Graph Store request operates on.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export type Target =
+  | { readonly _tag: 'Default' }
+  | { readonly _tag: 'Named'; readonly graph: Term.NamedNode }
+  /** The graph store itself (a `POST` creates a new graph). */
+  | { readonly _tag: 'Store' };
+
+/**
+ * Determines the target graph of a request to the graph store whose root URL
+ * is `root`.
+ *
+ * @since 0.1.0
+ * @category decoding
+ */
+export const target = (
+  url: URL,
+  root: string,
+): Effect.Effect<Target, ProtocolErrors.ProtocolError> => {
+  const params = Protocol.searchParams(url.href);
+  const graphs = params.getAll('graph');
+  const isDefault = params.has('default');
+  if (isDefault && Arr.isReadonlyArrayNonEmpty(graphs)) {
+    return Effect.fail(
+      ProtocolErrors.badRequest("A request cannot have both 'default' and 'graph'"),
+    );
+  }
+  if (graphs.length > 1)
+    return Effect.fail(ProtocolErrors.badRequest("Exactly one 'graph' is allowed"));
+  if (isDefault) return Effect.succeed({ _tag: 'Default' });
+  const graph = graphs[0];
+  if (graph !== undefined) {
+    const iri = Iri.resolve(graph, url.href);
+    if (!Iri.isAbsolute(iri) || /[\s<>"{}|^`\\]/.test(iri)) {
+      return Effect.fail(ProtocolErrors.badRequest(`Invalid graph IRI: ${graph}`));
+    }
+    return Effect.succeed({ _tag: 'Named', graph: Term.namedNode(iri) });
+  }
+  const direct = new URL(url.href);
+  direct.search = '';
+  direct.hash = '';
+  const rootUrl = new URL(root, url);
+  if (Options.requestPath(direct.pathname) === Options.requestPath(rootUrl.pathname)) {
+    return Effect.succeed({ _tag: 'Store' });
+  }
+  return Effect.succeed({ _tag: 'Named', graph: Term.namedNode(direct.href) });
+};
+
+const graphOf = (t: Target): Term.Graph =>
+  Predicate.isTagged(t, 'Named') ? t.graph : Term.defaultGraph;
+
+const describeTarget = (t: Target): string =>
+  Predicate.isTagged(t, 'Named') ? `<${t.graph.value}>` : 'the default graph';
+
+const storeError = (e: Store.StoreError) => ProtocolErrors.internal(e.message);
+
+const exists = (store: Store.QuadStoreShape, t: Target) =>
+  Predicate.isTagged(t, 'Named')
+    ? Effect.mapError(store.hasGraph(t.graph), storeError)
+    : Effect.succeed(true);
+
+const charsetUtf8 = (mediaType: string) => `${mediaType}; charset=utf-8`;
+
+const parseDocument = (
+  formats: Serialization.Formats['Service'],
+  contentType: string,
+  text: string,
+  baseIri: string,
+): Effect.Effect<ReadonlyArray<Term.Quad>, ProtocolErrors.ProtocolError> => {
+  const ct = Protocol.parseContentType(contentType);
+  if (ct === undefined) {
+    return Effect.fail(ProtocolErrors.unsupportedMediaType('Missing Content-Type'));
+  }
+  if (Protocol.isNonUtf8(ct)) {
+    return Effect.fail(
+      ProtocolErrors.unsupportedMediaType(`Unsupported charset '${ct.parameters['charset']}'`),
+    );
+  }
+  const parser = Serialization.parserFor(formats, ct.mediaType);
+  if (Option.isNone(parser)) {
+    return Effect.fail(
+      ProtocolErrors.unsupportedMediaType(
+        `Unsupported Content-Type '${ct.mediaType}'; supported: ${formats.parsers.map((p) => p.format.mediaType).join(', ')}`,
+      ),
+    );
+  }
+  return parser.value.parse(text, { baseIri }).pipe(
+    Effect.map((r) => Array.from(r.dataset)),
+    Effect.mapError((e) => ProtocolErrors.badRequest(Serialization.formatError(e))),
+  );
+};
+
+const partContentType = (
+  formats: Serialization.Formats['Service'],
+  contentType: string,
+  name: string,
+): string => {
+  const ct = Protocol.parseContentType(contentType);
+  if (ct !== undefined && Option.isSome(Serialization.parserFor(formats, ct.mediaType))) {
+    return contentType;
+  }
+  const dot = name.lastIndexOf('.');
+  if (dot >= 0) {
+    const parser = Serialization.parserForExtension(formats, name.slice(dot + 1));
+    if (Option.isSome(parser)) return parser.value.format.mediaType;
+  }
+  return contentType;
+};
+
+const readQuads = (
+  request: HttpServerRequest.HttpServerRequest,
+  formats: Serialization.Formats['Service'],
+  baseIri: string,
+): Effect.Effect<ReadonlyArray<Term.Quad>, ProtocolErrors.ProtocolError> =>
+  Effect.gen(function* () {
+    const contentType = request.headers['content-type'];
+    const ct = Protocol.parseContentType(contentType);
+    if (ct !== undefined && ct.mediaType === 'multipart/form-data') {
+      const parts = yield* Stream.runCollect(request.multipartStream).pipe(
+        Effect.mapError((e) => ProtocolErrors.badRequest(`Invalid multipart body: ${e.message}`)),
+      );
+      const perPart = yield* Effect.forEach(
+        parts,
+        Effect.fnUntraced(function* (part) {
+          if (Predicate.isTagged(part, 'Field')) {
+            if (part.value.trim() === '') return [];
+            return yield* parseDocument(
+              formats,
+              partContentType(formats, part.contentType, part.key),
+              part.value,
+              baseIri,
+            );
+          }
+          if (Predicate.isTagged(part, 'File')) {
+            const bytes = yield* part.contentEffect.pipe(
+              Effect.mapError((e) =>
+                ProtocolErrors.badRequest(`Invalid multipart body: ${e.message}`),
+              ),
+            );
+            return yield* parseDocument(
+              formats,
+              partContentType(formats, part.contentType, part.name),
+              new TextDecoder().decode(bytes),
+              baseIri,
+            );
+          }
+          return [];
+        }),
+        { concurrency: 1 },
+      );
+      return perPart.flat();
+    }
+    const text = yield* Effect.mapError(request.text, (e) =>
+      ProtocolErrors.badRequest(`Could not read the request body: ${e.message}`),
+    );
+    if (contentType === undefined && text.trim() === '') return [];
+    return yield* parseDocument(formats, contentType ?? '', text, baseIri);
+  });
+
+const intoGraph = (quads: ReadonlyArray<Term.Quad>, graph: Term.Graph): Array<Term.Quad> =>
+  quads.map((q) => new Term.Quad(q.subject, q.predicate, q.object, graph));
+
+/**
+ * Handles a Graph Store Protocol request. `root` is the path of the graph
+ * store.
+ *
+ * @since 0.1.0
+ * @category handlers
+ */
+export const handle = (
+  options: Options.Resolved,
+  root: string,
+): Effect.Effect<
+  HttpServerResponse.HttpServerResponse,
+  ProtocolErrors.ProtocolError,
+  HttpServerRequest.HttpServerRequest | Store.QuadStore | Serialization.Formats
+> =>
+  Effect.gen(function* () {
+    const request = yield* HttpServerRequest.HttpServerRequest;
+    const store = yield* Store.QuadStore;
+    const formats = yield* Serialization.Formats;
+    const url = Options.requestUrl(request, options);
+    const t = yield* target(url, root);
+    const method = request.method;
+    const writes = ['PUT', 'POST', 'DELETE'];
+    const allowed = options.readOnly
+      ? ['GET', 'HEAD', 'OPTIONS']
+      : Predicate.isTagged(t, 'Store')
+        ? ['POST', 'OPTIONS']
+        : ['GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'OPTIONS'];
+    if (options.readOnly && writes.includes(method)) {
+      return yield* ProtocolErrors.methodNotAllowed(method, allowed);
+    }
+    yield* Effect.annotateCurrentSpan({ 'sparql.graph': describeTarget(t) });
+
+    // Runs a write in one transaction and answers 204 if the graph existed, 201 if it is new.
+    const commit = (write: Effect.Effect<void, Store.StoreError>, existed: boolean) =>
+      store
+        .transaction(write)
+        .pipe(
+          Effect.mapError(storeError),
+          Effect.as(HttpServerResponse.empty({ status: existed ? 204 : 201 })),
+        );
+    return yield* Match.value(method).pipe(
+      Match.when(Match.is('GET', 'HEAD'), () =>
+        Effect.gen(function* () {
+          if (Predicate.isTagged(t, 'Store')) {
+            return yield* ProtocolErrors.badRequest(
+              "Identify a graph with '?default', '?graph=<iri>' or a URL below the graph store",
+            );
+          }
+          if (!(yield* exists(store, t))) {
+            return yield* ProtocolErrors.notFound(`No such graph: ${describeTarget(t)}`);
+          }
+          const accept = request.headers['accept'];
+          const serializer = ServerFormats.negotiateSerializer(formats, accept);
+          if (serializer === undefined) {
+            return yield* ProtocolErrors.notAcceptable(
+              accept ?? '',
+              ServerFormats.serializerMediaTypes(formats),
+            );
+          }
+          const headers = { 'content-type': charsetUtf8(serializer.format.mediaType) };
+          if (method === 'HEAD') return HttpServerResponse.empty({ status: 200, headers });
+          const quads = Stream.map(
+            store.match({ graph: graphOf(t) }),
+            (q) => new Term.Quad(q.subject, q.predicate, q.object, Term.defaultGraph),
+          );
+          const body = serializer.serializeStream(quads, {
+            prefixes: PrefixMap.make(options.prefixes),
+          });
+          return HttpServerResponse.stream(Stream.encodeText(body), {
+            status: 200,
+            headers,
+            contentType: charsetUtf8(serializer.format.mediaType),
+          });
+        }),
+      ),
+      Match.when('PUT', () =>
+        Effect.gen(function* () {
+          if (Predicate.isTagged(t, 'Store'))
+            return yield* ProtocolErrors.methodNotAllowed(method, allowed);
+          const base = Predicate.isTagged(t, 'Named') ? t.graph.value : url.href;
+          const quads = yield* readQuads(request, formats, base);
+          const existed = Predicate.isTagged(t, 'Named') ? yield* exists(store, t) : true;
+          return yield* commit(
+            Effect.gen(function* () {
+              if (Predicate.isTagged(t, 'Named')) {
+                yield* store.dropGraph(t.graph);
+                yield* store.createGraph(t.graph);
+              } else {
+                yield* store.clear(Term.defaultGraph);
+              }
+              yield* store.insert(intoGraph(quads, graphOf(t)));
+            }),
+            existed,
+          );
+        }),
+      ),
+      Match.when('POST', () =>
+        Effect.gen(function* () {
+          if (Predicate.isTagged(t, 'Store')) {
+            const id = globalThis.crypto.randomUUID();
+            const rootUrl = new URL(root, url);
+            const graph = Term.namedNode(`${rootUrl.href.replace(/\/$/, '')}/${id}`);
+            const quads = yield* readQuads(request, formats, graph.value);
+            yield* store
+              .transaction(
+                Effect.andThen(store.createGraph(graph), store.insert(intoGraph(quads, graph))),
+              )
+              .pipe(Effect.mapError(storeError));
+            return HttpServerResponse.empty({ status: 201, headers: { location: graph.value } });
+          }
+          const base = Predicate.isTagged(t, 'Named') ? t.graph.value : url.href;
+          const quads = yield* readQuads(request, formats, base);
+          const existed = yield* exists(store, t);
+          return yield* commit(
+            Effect.gen(function* () {
+              if (Predicate.isTagged(t, 'Named') && !existed) yield* store.createGraph(t.graph);
+              yield* store.insert(intoGraph(quads, graphOf(t)));
+            }),
+            existed,
+          );
+        }),
+      ),
+      Match.when('DELETE', () =>
+        Effect.gen(function* () {
+          if (Predicate.isTagged(t, 'Store'))
+            return yield* ProtocolErrors.methodNotAllowed(method, allowed);
+          if (!Predicate.isTagged(t, 'Named')) {
+            yield* Effect.mapError(store.clear(Term.defaultGraph), storeError);
+            return HttpServerResponse.empty({ status: 204 });
+          }
+          if (!(yield* exists(store, t))) {
+            return yield* ProtocolErrors.notFound(`No such graph: ${describeTarget(t)}`);
+          }
+          yield* Effect.mapError(store.dropGraph(t.graph), storeError);
+          return HttpServerResponse.empty({ status: 204 });
+        }),
+      ),
+      Match.orElse(() =>
+        Effect.gen(function* () {
+          return yield* ProtocolErrors.methodNotAllowed(method, allowed);
+        }),
+      ),
+    );
+  }).pipe(Effect.withSpan('SparqlServer.graphStore'));

package/src/sparql/Optimizer.ts

@@ -0,0 +1,230 @@
+/**
+ * A simple, semantics-preserving algebra optimizer:
+ *
+ * - **Filter pushdown**: conjunctive filter conditions are pushed below
+ *   `Join` and into the left side of `LeftJoin` / `Minus` when all their
+ *   variables are certainly bound there (and they contain no `EXISTS`).
+ * - **BGP join ordering**: the triple patterns of each BGP are reordered
+ *   greedily by estimated selectivity (using `QuadStore.estimate` when the
+ *   store provides it, and a static heuristic otherwise), preferring
+ *   patterns connected to already-bound variables.
+ *
+ * Patterns inside `EXISTS` are left untouched.
+ *
+ * @since 0.1.0
+ */
+import * as Arr from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Match from 'effect/Match';
+import * as Option from 'effect/Option';
+import * as Predicate from 'effect/Predicate';
+import * as Term from '../Term.ts';
+import * as Algebra from './Algebra.ts';
+import type * as Ast from './Ast.ts';
+import { loop } from './internal/loop.ts';
+import * as StringMap from './internal/stringMap.ts';
+import type * as Store from './Store.ts';
+
+/**
+ * Inputs of the optimizer.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface OptimizerContext {
+  readonly store: Store.QuadStoreShape;
+  readonly dataset: Store.DatasetSpec;
+  readonly graph: Store.ActiveGraph;
+}
+
+const conjuncts = (e: Algebra.Expression): Array<Algebra.Expression> =>
+  Predicate.isTagged(e, 'Operation') && e.operator === '&&' ? e.args.flatMap(conjuncts) : [e];
+
+const conjunction = (es: ReadonlyArray<Algebra.Expression>): Algebra.Expression =>
+  es.reduce((a, b) => ({ _tag: 'Operation', operator: '&&', args: [a, b] }));
+
+const subset = (a: ReadonlySet<string>, b: ReadonlySet<string>): boolean =>
+  Array.from(a).every((x) => b.has(x));
+
+/** Pushes one filter condition as deep as possible; `None` if it cannot move. */
+const push = (e: Algebra.Expression, op: Algebra.Operation): Option.Option<Algebra.Operation> => {
+  const vars = Algebra.expressionVariables(e);
+  return Match.value(op).pipe(
+    Match.tag('Join', (j): Option.Option<Algebra.Operation> => {
+      if (subset(vars, Algebra.certainVariables(j.left))) {
+        return Option.some({ ...j, left: pushOrWrap(e, j.left) });
+      }
+      if (subset(vars, Algebra.certainVariables(j.right))) {
+        return Option.some({ ...j, right: pushOrWrap(e, j.right) });
+      }
+      return Option.none();
+    }),
+    Match.tag('LeftJoin', 'Minus', (o) =>
+      subset(vars, Algebra.certainVariables(o.left))
+        ? Option.some({ ...o, left: pushOrWrap(e, o.left) })
+        : Option.none(),
+    ),
+    Match.tag('Filter', (f) => Option.some({ ...f, input: pushOrWrap(e, f.input) })),
+    Match.orElse(() => Option.none()),
+  );
+};
+
+const pushOrWrap = (e: Algebra.Expression, op: Algebra.Operation): Algebra.Operation =>
+  Option.getOrElse(push(e, op), () => Algebra.filter(e, op));
+
+const pushFilters = (op: Algebra.Operation): Algebra.Operation =>
+  Algebra.transform(op, (o) => {
+    if (!Predicate.isTagged(o, 'Filter')) return o;
+    // The conditions that stay, and the input with the others pushed in.
+    const kept: Array<Algebra.Expression> = [];
+    const keep = (input: Algebra.Operation, c: Algebra.Expression): Algebra.Operation => {
+      kept.push(c);
+      return input;
+    };
+    const input = conjuncts(o.expression).reduce(
+      (acc, c) =>
+        Algebra.containsExists(c)
+          ? keep(acc, c)
+          : Option.getOrElse(push(c, acc), () => keep(acc, c)),
+      o.input,
+    );
+    return Arr.isReadonlyArrayEmpty(kept) ? input : Algebra.filter(conjunction(kept), input);
+  });
+
+const positions = (p: Ast.TriplePattern): ReadonlyArray<Ast.TermPattern> => [
+  p.subject,
+  p.predicate,
+  p.object,
+];
+
+const patternVariables = (p: Ast.TriplePattern): ReadonlyArray<string> => {
+  const out: Array<string> = [];
+  const go = (t: Ast.TermPattern | Term.Term): void => {
+    if (Term.isVariable(t)) out.push(t.value);
+    else if (Term.isTriple(t)) {
+      go(t.subject);
+      go(t.predicate);
+      go(t.object);
+    }
+  };
+  positions(p).forEach(go);
+  return out;
+};
+
+/** A pattern term as a constant: `None` for a variable or a triple pattern with variables. */
+const constant = (t: Ast.TermPattern): Option.Option<Term.GroundTerm> => {
+  if (Term.isVariable(t)) return Option.none();
+  if (Term.isTriple(t)) {
+    const vars = patternVariables({
+      _tag: 'TriplePattern',
+      subject: t.subject,
+      predicate: t.predicate,
+      object: t.object,
+    });
+    return Arr.isReadonlyArrayNonEmpty(vars) ? Option.none() : Option.some(t);
+  }
+  return Option.some(t);
+};
+
+/** Static selectivity heuristic when the store gives no estimates. */
+const staticCost = (p: Ast.TriplePattern): number =>
+  (Option.isNone(constant(p.subject)) ? 100 : 1) *
+  (Option.isNone(constant(p.object)) ? 10 : 1) *
+  (Option.isNone(constant(p.predicate)) ? 5 : 1);
+
+const orderBgp = (
+  patterns: ReadonlyArray<Ast.TriplePattern>,
+  ctx: OptimizerContext,
+): Effect.Effect<ReadonlyArray<Ast.TriplePattern>, Store.StoreError> =>
+  Effect.gen(function* () {
+    if (patterns.length < 2) return patterns;
+    const estimate = ctx.store.estimate;
+    const costs = yield* Effect.forEach(
+      patterns,
+      (p) =>
+        estimate === undefined
+          ? Effect.succeed(staticCost(p))
+          : estimate({
+              subject: Option.getOrUndefined(constant(p.subject)),
+              predicate: Option.getOrUndefined(constant(p.predicate)),
+              object: Option.getOrUndefined(constant(p.object)),
+              graph: Predicate.isTagged(ctx.graph, 'NamedGraph')
+                ? ctx.graph.graph
+                : ctx.dataset.defaultGraph === 'default'
+                  ? Term.defaultGraph
+                  : undefined,
+            }),
+      { concurrency: 1 },
+    );
+    const remaining = Arr.zip(patterns, costs).map(([p, cost]) => ({
+      p,
+      cost,
+      vars: patternVariables(p),
+    }));
+    const bound = StringMap.emptySet();
+    const out: Array<Ast.TriplePattern> = [];
+    // Greedily picks the cheapest pattern, preferring those connected to the bound variables.
+    loop(() => {
+      if (!Arr.isArrayNonEmpty(remaining)) return false;
+      const connected = remaining.some((r) => r.vars.some((v) => bound.has(v)));
+      const scored = Arr.map(remaining, (r, i) => {
+        const isConnected = r.vars.some((v) => bound.has(v));
+        const eligible = !(connected && !isConnected && bound.size > 0);
+        const boundCount = r.vars.filter((v) => bound.has(v)).length;
+        return { r, i, score: eligible ? r.cost / 10 ** boundCount : Infinity };
+      });
+      // The first with the lowest score (the first pattern if none is eligible).
+      const best = scored.reduce((a, b) => (b.score < a.score ? b : a));
+      remaining.splice(best.i, 1);
+      out.push(best.r.p);
+      best.r.vars.forEach((v) => bound.set(v, true));
+      return true;
+    });
+    return out;
+  });
+
+const orderBgps = (
+  op: Algebra.Operation,
+  ctx: OptimizerContext,
+): Effect.Effect<Algebra.Operation, Store.StoreError> => {
+  return Match.value(op).pipe(
+    Match.tag('Bgp', (b): Effect.Effect<Algebra.Operation, Store.StoreError> =>
+      Effect.map(orderBgp(b.patterns, ctx), (patterns) => Algebra.bgp(patterns)),
+    ),
+    Match.tag('Path', 'Values', (o) => Effect.succeed(o)),
+    Match.tag('Graph', (g) =>
+      Effect.map(
+        orderBgps(g.input, {
+          ...ctx,
+          graph: Term.isNamedNode(g.name) ? { _tag: 'NamedGraph', graph: g.name } : ctx.graph,
+        }),
+        (input) => ({ ...g, input }),
+      ),
+    ),
+    Match.orElse((o) => {
+      const children: Array<Algebra.Operation> = [];
+      Algebra.mapChildren(o, (c) => {
+        children.push(c);
+        return c;
+      });
+      return Effect.map(
+        Effect.forEach(children, (c) => orderBgps(c, ctx), { concurrency: 1 }),
+        (optimized) => {
+          let i = 0;
+          return Algebra.mapChildren(o, (c) => optimized[i++] ?? c);
+        },
+      );
+    }),
+  );
+};
+
+/**
+ * Optimizes an algebra expression.
+ *
+ * @since 0.1.0
+ * @category optimization
+ */
+export const optimize = (
+  op: Algebra.Operation,
+  ctx: OptimizerContext,
+): Effect.Effect<Algebra.Operation, Store.StoreError> => orderBgps(pushFilters(op), ctx);