@leuduan/block 2.0.0 → 3.1.0

package/README.md

@@ -12,22 +12,28 @@ renderer (`l`, 120 columns by default) and the builders on top of it: `p` (pipes
 helpers), `q` (queries), `t` (table expressions) and the `sql` tag, with `from` exported at the root. Every function
 has one name and one home; the namespaces export factories, not the concrete classes behind them. A pipe is applied
 one way: `pipe(query, p.where(...), ...)`. Pipe syntax is GoogleSQL's, so the result is a `GoogleSqlQuery`; applying
-pipes to one extends its own pipe list instead of nesting (reusable steps are a `Pipe[]` to spread into the call).
+pipes to one extends its own pipe list instead of nesting. A step can also be a function from the query built so far
+to the next query, for what `|>` cannot express (the input on the right of a join, inside a subquery, or rewritten);
+later steps apply to what it returns. Reusable steps are a `Pipe[]` to spread into the call, or such a function, with
+any inputs taken as arguments of the function that makes them.
 Deliberate nesting is `q.googleQuery({ queryExpression, pipes })`. Pipes that take a list take an array. String
 literals (`e.literalString`, the typed date and time literals) and quoted identifiers (`e.ident`, `t.path(id, true)`)
 are escaped with GoogleSQL's backslash escapes.
 
 It also owns datasets: `define` declares one (a table, or a query whose `build` receives a `BuildContext`) and
-records its call site, `problemsOf` judges a definition, and `datasetBuilder` builds them. A builder is one session
-that builds each (dataset, case) once: it resolves parameter values from defaults and cases, keeps one value per
-parameter name across inlined queries, inlines references and refuses cycles, and returns the result graph
+records its call site, `problemsOf` judges a definition, and `datasetBuilder` builds them. Every definition carries
+its own `id`, a path of names from the outermost namespace to the dataset (`['analytics', 'events']`), so a dataset
+keeps its name wherever its declaration lives; messages show it dotted, and no segment may contain a dot. A builder is
+one session that builds each
+(dataset, case) once: it refuses two datasets with one id, resolves parameter values from defaults and cases, keeps
+one value per parameter name across inlined queries, inlines references and refuses cycles, and returns the result graph
 (`BuiltDataset`: the query, its parameters, the results it referenced, the assets it loaded). A failure inside a
 referenced dataset is rethrown at the `ctx.ref` call with the original as its `cause`.
 
 It does not own table names, projects, asset files or what a query means. The builder takes them from the caller, with
-no defaults: `idOf` names a dataset in messages (and refuses one the caller does not know), `tableName` gives a table's
-fully-qualified name, `asset` loads an asset's SQL text. Policies over the built SQL, such as a rule
-against handwritten table ids, are the caller's too. Quoting is the caller's job as well: `from(string)` and `t.path(id, false)` emit the text as given, so use `t.path(id, true)` or
+no defaults: `tableName` gives a table's fully-qualified name, `asset` loads an asset's SQL text. Policies over the
+built SQL, such as a rule against handwritten table ids or references outside a catalog (the result graph lists
+them), are the caller's too. Quoting is the caller's job as well: `from(string)` and `t.path(id, false)` emit the text as given, so use `t.path(id, true)` or
 `e.ident(name)` for identifiers. `create_table` and the ANSI dialect (`dbox.block.ansi`) are not ported.
 
 ## Use
@@ -49,28 +55,19 @@ duplicates.format() // FROM `my-project.analytics.events` |> SELECT ...
 Datasets, built standalone:
 
 ```ts
-import { datasetBuilder, define, p, pipe, sql } from '@leuduan/block'
+import { Dataset, datasetBuilder, define, p, pipe, sql } from '@leuduan/block'
 
-const EVENTS = define({ kind: 'table', description: 'Raw events.', grain: 'one row per event' })
+const EVENTS = define({ kind: 'table', id: ['analytics', 'events'], description: 'Raw events.' })
 const recent = define({
   kind: 'query',
+  id: ['analytics', 'recent_events'],
   description: 'Events since a day.',
-  grain: 'one row per event',
   params: { day: { type: 'DATE', default: '2026-01-01' } },
   build: (ctx) => pipe(ctx.ref(EVENTS), p.where(sql`day >= ${ctx.param('day')}`)),
 })
 
-const names = new Map([
-  [EVENTS, 'events'],
-  [recent, 'recent'],
-])
 const builder = datasetBuilder({
-  idOf: (dataset) => {
-    const name = names.get(dataset)
-    if (name === undefined) throw new Error('not a dataset of this catalog')
-    return name
-  },
-  tableName: () => 'my-project.analytics.events',
+  tableName: (dataset) => `my-project.${Dataset.definitionOf(dataset).id.join('.')}`,
   asset: (file) => {
     throw new Error(`no assets here: ${file}`)
   },

package/dist/build.d.ts

@@ -8,11 +8,6 @@ export interface Parameter {
     value: ParamValue;
 }
 export interface BuildOptions {
-    /**
-     * The caller's id for a dataset, used in messages. It throws for a dataset the caller does not know, which refuses
-     * a `ctx.ref` to it.
-     */
-    idOf(dataset: Dataset): string;
     /** The fully-qualified name of a table dataset. */
     tableName(dataset: Dataset): string;
     /** The SQL text of the asset `file` that `dataset`'s build asks for. */
@@ -35,7 +30,8 @@ export interface BuiltDataset {
 export interface DatasetBuilder {
     /**
      * `dataset` built for `caseName` (`null` builds with the parameter defaults). Throws what stops the build: a
-     * blocking problem of a definition, a missing parameter value, a reference cycle, an error of a `build`. A failure
+     * blocking problem of a definition, an id another dataset of this session already has, a missing parameter value, a
+     * reference cycle, an error of a `build`. A failure
      * inside a referenced dataset is rethrown at the `ctx.ref` call, its error as the `cause`.
      */
     build(dataset: Dataset, caseName: string | null): BuiltDataset;

package/dist/build.js

@@ -7,6 +7,10 @@ import { problemsOf } from './problems.js';
 import { raw } from './query.js';
 import { QueryExpression, from } from './sql.js';
 import { path } from './t.js';
+/** The dotted form of a dataset's id: one string per dataset, since no segment contains a dot. */
+function idOf(dataset) {
+    return Dataset.definitionOf(dataset).id.join('.');
+}
 function messageOf(error) {
     return error instanceof Error ? error.message : String(error);
 }
@@ -32,6 +36,7 @@ function mergeParameters(into, parameters, origin) {
 }
 export function datasetBuilder(options) {
     const results = new Map();
+    const datasetsById = new Map();
     const active = [];
     function cached(dataset, caseName) {
         return results.get(dataset)?.get(Dataset.definitionOf(dataset).kind === 'table' ? null : caseName);
@@ -39,7 +44,7 @@ export function datasetBuilder(options) {
     /** `loop` runs from a reference's target to a dataset still being built. */
     function cycleError(loop) {
         const start = active.indexOf(loop[loop.length - 1]);
-        return new Error(`reference cycle: ${[...active.slice(start), ...loop].map(options.idOf).join(' -> ')}`);
+        return new Error(`reference cycle: ${[...active.slice(start), ...loop].map(idOf).join(' -> ')}`);
     }
     /**
      * The path from a result built earlier down to a dataset being built now, if there is one. A build that caught a
@@ -73,7 +78,7 @@ export function datasetBuilder(options) {
             ref(target, refOptions) {
                 if (!(target instanceof Dataset))
                     throw new Error('ctx.ref needs a dataset');
-                const id = options.idOf(target);
+                const id = idOf(target);
                 if (active.includes(target))
                     throw cycleError([target]);
                 const targetCase = refOptions?.case ?? null;
@@ -129,6 +134,11 @@ export function datasetBuilder(options) {
         const blocking = problemsOf(definition).find((problem) => problem.blocks);
         if (blocking)
             throw new Error(blocking.message);
+        const id = idOf(dataset);
+        const sameId = datasetsById.get(id);
+        if (sameId && sameId !== dataset)
+            throw new Error(`two datasets have the id "${id}"`);
+        datasetsById.set(id, dataset);
         let result;
         active.push(dataset);
         try {

package/dist/descriptor.d.ts

@@ -9,18 +9,20 @@ export interface ParamSpec {
 }
 export type ParamsSpec = Record<string, ParamSpec>;
 interface CommonDefinition {
+    /**
+     * Names the dataset wherever it is declared, so moving the declaration does not rename it. Segments run from the
+     * outermost namespace to the name; none contains a dot, so the dotted form in messages names one dataset.
+     */
+    id: readonly [string, ...string[]];
     description: string;
-    grain: string;
     /** Notes on what columns mean. Free text, never a schema. */
     columns?: Record<string, string>;
-    /** Notes on how this dataset joins others, keyed by the other dataset's name. */
-    relations?: Record<string, string>;
 }
 export interface TableDefinition extends CommonDefinition {
     kind: 'table';
     /** Physical table name, for the builder's `tableName` to use. */
     table?: string;
-    partitionedBy?: string;
+    partitionedBy?: readonly string[];
     clusteredBy?: readonly string[];
 }
 export interface BuildContext<Name extends string = string> {

package/dist/index.d.ts

@@ -6,7 +6,7 @@ export * as q from './query.js';
 export * as t from './t.js';
 export type { Line, LineOrStr, Options } from './line.js';
 export { DEFAULT_OPTIONS } from './line.js';
-export type { SqlSegment, Pipe, TableExpression, GoogleSqlQuery } from './sql.js';
+export type { SqlSegment, Pipe, PipeStep, TableExpression, GoogleSqlQuery } from './sql.js';
 export { QueryExpression, from, pipe, sql } from './sql.js';
 export { define } from './define.js';
 export { Dataset, PARAM_TYPES } from './descriptor.js';

package/dist/problems.js

@@ -46,12 +46,13 @@ export function problemsOf(def) {
     }
     const out = [];
     const flag = (message, blocks = false) => out.push({ message, blocks });
+    const id = def.id;
+    if (!(Array.isArray(id) && id.length > 0 && id.every((segment) => nonEmpty(segment) && !segment.includes('.')))) {
+        flag('`id` is a non-empty list of names without dots', true);
+    }
     if (!nonEmpty(def.description))
         flag('`description` is missing or empty');
-    if (!nonEmpty(def.grain))
-        flag('`grain` is missing or empty');
     notes('columns', def.columns, out);
-    notes('relations', def.relations, out);
     if (def.kind === 'table') {
         for (const field of ['build', 'params', 'cases', 'role']) {
             if (def[field] !== undefined)
@@ -60,11 +61,11 @@ export function problemsOf(def) {
         if (def.table !== undefined && !(typeof def.table === 'string' && TABLE_NAME.test(def.table))) {
             flag('`table` is not a valid table name', true);
         }
-        if (def.partitionedBy !== undefined && !nonEmpty(def.partitionedBy))
-            flag('`partitionedBy` is a column name');
-        const clustered = def.clusteredBy;
-        if (clustered !== undefined && !(Array.isArray(clustered) && clustered.every(nonEmpty))) {
-            flag('`clusteredBy` is a list of column names');
+        for (const field of ['partitionedBy', 'clusteredBy']) {
+            const columns = def[field];
+            if (columns !== undefined && !(Array.isArray(columns) && columns.every(nonEmpty))) {
+                flag(`\`${field}\` is a list of column names`);
+            }
         }
         return out;
     }

package/dist/sql.d.ts

@@ -53,12 +53,15 @@ export declare class GoogleSqlQuery extends QueryExpression {
     constructor({ ctes, queryExpression, orderByLine, limitOffset, pipes }: GoogleSqlQueryProps);
     toLine(): Line;
 }
+/** A step of `pipe`: a `|>` operator, or a function from the query built so far to the next query. */
+export type PipeStep = Pipe | ((query: QueryExpression) => QueryExpression);
 /**
- * Apply `|>` steps to a query. Pipe syntax is GoogleSQL's, so the result is a GoogleSqlQuery. A GoogleSqlQuery takes
- * the steps on its own pipe list (only its outer wrapper is flattened); anything else is wrapped. The input is not
- * changed.
+ * Apply steps to a query, left to right. `|>` operators are GoogleSQL's pipe syntax, so a call of only those returns a
+ * GoogleSqlQuery; a function step gets the query built so far and its result is what later steps apply to. The input
+ * is not changed.
  */
 export declare function pipe(query: QueryExpression, ...steps: Pipe[]): GoogleSqlQuery;
+export declare function pipe(query: QueryExpression, ...steps: PipeStep[]): QueryExpression;
 type SqlInterp = string | number | Line | SqlSegment;
 /**
  * Tagged template for raw SQL fragments.

package/dist/sql.js

@@ -119,22 +119,32 @@ export class GoogleSqlQuery extends QueryExpression {
         ]);
     }
 }
-/**
- * Apply `|>` steps to a query. Pipe syntax is GoogleSQL's, so the result is a GoogleSqlQuery. A GoogleSqlQuery takes
- * the steps on its own pipe list (only its outer wrapper is flattened); anything else is wrapped. The input is not
- * changed.
- */
-export function pipe(query, ...steps) {
+/** A GoogleSqlQuery takes the pipes on its own pipe list (only its outer wrapper is flattened); anything else is wrapped. */
+function appendPipes(query, pipes) {
     if (query instanceof GoogleSqlQuery) {
         return new GoogleSqlQuery({
             ctes: query.ctes,
             queryExpression: query.queryExpression,
             orderByLine: query.orderByLine,
             limitOffset: query.limitOffset,
-            pipes: [...query.pipes, ...steps],
+            pipes: [...query.pipes, ...pipes],
         });
     }
-    return new GoogleSqlQuery({ queryExpression: query, pipes: steps });
+    return new GoogleSqlQuery({ queryExpression: query, pipes });
+}
+export function pipe(query, ...steps) {
+    let result = query;
+    let pending = [];
+    for (const step of steps) {
+        if (step instanceof Pipe) {
+            pending.push(step);
+            continue;
+        }
+        result = step(pending.length > 0 ? appendPipes(result, pending) : result);
+        pending = [];
+    }
+    const endsWithFunction = steps.length > 0 && !(steps[steps.length - 1] instanceof Pipe);
+    return endsWithFunction ? result : appendPipes(result, pending);
 }
 /**
  * Tagged template for raw SQL fragments.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leuduan/block",
-  "version": "2.0.0",
+  "version": "3.1.0",
   "description": "SQL builder and renderer for GoogleSQL",
   "license": "ISC",
   "author": "Duan Leu",