@malloydata/malloyyo 0.2.54 → 0.2.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +951 -389
- package/dist/shared/nav.ts +222 -30
- package/dist/templates/skills/malloyyo-datasets/SKILL.md +35 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
4
|
import { Command } from "commander";
|
|
5
|
-
import { resolve as resolve4 } from "node:path";
|
|
5
|
+
import { resolve as resolve4, relative as relative2, sep as sep2 } from "node:path";
|
|
6
6
|
|
|
7
7
|
// src/config.ts
|
|
8
8
|
import { readFileSync, existsSync } from "node:fs";
|
|
@@ -172,19 +172,14 @@ function resolvePublishTarget(dir, name, overrides = {}) {
|
|
|
172
172
|
}
|
|
173
173
|
|
|
174
174
|
// src/gather.ts
|
|
175
|
-
import { readFileSync as readFileSync2, existsSync as existsSync2, readdirSync, statSync } from "node:fs";
|
|
176
|
-
import { join as join2, relative, sep } from "node:path";
|
|
175
|
+
import { readFileSync as readFileSync2, existsSync as existsSync2, readdirSync, statSync, mkdtempSync, rmSync } from "node:fs";
|
|
176
|
+
import { join as join2, relative, sep, dirname } from "node:path";
|
|
177
|
+
import { tmpdir } from "node:os";
|
|
177
178
|
import { execFileSync } from "node:child_process";
|
|
178
179
|
|
|
179
|
-
// src/
|
|
180
|
-
import
|
|
181
|
-
import
|
|
182
|
-
import url3 from "node:url";
|
|
183
|
-
import {
|
|
184
|
-
MalloyConfig,
|
|
185
|
-
Runtime,
|
|
186
|
-
discoverConfig
|
|
187
|
-
} from "@malloydata/malloy";
|
|
180
|
+
// src/repo.ts
|
|
181
|
+
import fs4 from "node:fs";
|
|
182
|
+
import path5 from "node:path";
|
|
188
183
|
|
|
189
184
|
// ../mcp-engine/dist/index.js
|
|
190
185
|
import {
|
|
@@ -215,7 +210,9 @@ var contentFiles = {
|
|
|
215
210
|
"explore/restricted-queries.md": "# Restricted Query Explanation\n\nThe `query` tool runs your Malloy against a **published model**. You have that\nmodel's entire published surface to work with \u2014 and you can build on it. The\nmodel is an inentionally curated subset of the data available in the\ndatabase.\n\n## You can\n\n- Use everything the model defines: its **sources, dimensions, measures, views,\n joins, and named queries**. `describe_source` shows exactly what's there.\n- **Run a named query and refine it** \u2014\n `run: top_carriers + { where: dep_year = 2024 }`.\n- **Define your own** dimensions, measures, and **your own sources and joins** \u2014\n as long as they are *derived from the model's sources*. You are not limited to\n the author's fields; compose new ones from them.\n- Reference the model's `$NAME` givens and supply values via the `givens` map on\n the `query` call (use `execute: false` to discover which a query needs).\n- Use a model field that was itself defined with raw SQL \u2014 the author vouched\n for the model's own definitions.\n\n## What is \"Restricted\"\n\nIf you see `restricted-construct-forbidden`, the query used something that\nreaches *outside* the published model: pulling in another file (`import`),\nopening a raw connection (`connection.table(...)` / `connection.sql(...)`),\nwriting raw SQL (`name!type(...)` or the `sql_*` functions), declaring new\n`given:`s, or setting `##!` compiler flags.\n\nThe fix is never to work around it \u2014 express the answer in terms of what the\nmodel publishes (define derived sources, joins, dimensions, and measures from\nthe model's sources). If something fundamental is missing, that's feedback for\nthe model's author.\n",
|
|
216
211
|
"language/malloy-language-reference.md": '<!-- Copied from malloy-cli (jrtipton/malloy-cli) skills/malloy-language-reference.md on 2026-06-11.\n Deliberate temporary fork \u2014 converge when the engine is extracted to @malloydata. -->\n---\ndescription: Malloy language reference \u2014 concepts, syntax, compilation model. Load this before writing or reviewing Malloy code.\n---\n# Malloy Language Reference\n\nMalloy is a semantic data modeling and query language. It compiles to SQL and runs against existing database engines (DuckDB, BigQuery, Snowflake, PostgreSQL, MySQL, Trino, Presto). It is not a SQL wrapper or abstraction layer \u2014 it has its own type system, scoping rules, expression semantics, and compilation pipeline.\n\nMalloy is designed around how humans think about data, not how data computations are mechanically accomplished. SQL is oriented around the machine \u2014 you specify joins, group-by columns, subqueries, and window functions in terms of what the database needs to do. Malloy is oriented around the analyst \u2014 you describe relationships, name computations, and compose questions in terms of what the data means. Malloy bridges the gap between these two by compiling the human-oriented description into correct, efficient SQL.\n\nA core design principle is that **most queries are themselves designing a new semantic model.** Formulating a question about data \u2014 choosing what to group by, what to aggregate, what to nest \u2014 is inherently an act of defining a new way to look at that data. Malloy is built around this idea: the output of every query is not just a result set but a new source with its own schema, and data comprehension is an ongoing iterative process where later stages want not only the data from a previous stage but how that data came into being. This is why query output carries metadata, why queries can be used as sources, and why views and pipelines compose naturally.\n\n## Documents and Statements\n\nA Malloy file (`.malloy`) is a sequence of statements, optionally separated by semicolons. There are five statement types:\n\n- **`import`** \u2014 import sources and queries from another `.malloy` file\n- **`source:`** \u2014 define a named, reusable data source with its schema and extensions\n- **`query:`** \u2014 define a named query (source + view) for reuse\n- **`run:`** \u2014 execute a query (the "do it now" statement)\n- **`given:`** \u2014 declare model-level parameters supplied at run time (experimental, see Givens)\n\n```malloy\nimport "shared_model.malloy"\n\nsource: flights is duckdb.table(\'flights.parquet\') extend {\n measure: flight_count is count()\n}\n\nquery: carrier_summary is flights -> {\n group_by: carrier\n aggregate: flight_count\n}\n\nrun: carrier_summary\n```\n\nComments use `//` or `--` (both are line comments).\n\n## Sources\n\nA **source** is anything you can hand a SQL database and get a schema back \u2014 a table name, a SQL SELECT, or the output of another Malloy query. The columns in that schema become the source\'s initial fields (all dimensions).\n\n```malloy\nsource: flights is duckdb.table(\'flights.parquet\')\nsource: limited is duckdb.sql("""SELECT * FROM flights LIMIT 100""")\nsource: carrier_facts is carrier_summary -- a query used as a source\n```\n\nWhat makes sources central to Malloy is **extension**. The `extend` block lets you layer on dimensions, measures, views, joins, filters, primary keys, field restrictions, and renames. These extensions travel with the source \u2014 any query against it gets them for free.\n\n```malloy\nsource: flights is duckdb.table(\'flights.parquet\') extend {\n primary_key: id\n\n dimension: distance_km is distance * 1.609344\n\n measure:\n flight_count is count()\n total_distance is sum(distance)\n\n join_one: carriers with carrier\n join_one: origin_airport is airports on origin_airport.code = origin\n\n where: dep_time > @2001\n\n view: by_carrier is {\n group_by: carrier\n aggregate: flight_count, total_distance\n }\n}\n```\n\nSources can extend other sources, creating a refinement chain:\n\n```malloy\nsource: ca_flights is flights extend {\n where: origin.state = \'CA\'\n}\n```\n\nField access control uses `accept:` (allowlist) or `except:` (denylist) to restrict which inherited columns are visible. Fields can be renamed with `rename: new_name is old_name`.\n\n## Joins\n\nJoins are declared in the source, not reconstructed in every query. This is a fundamental design difference from SQL: the graph structure of your data is a property of the model.\n\n```malloy\njoin_one: carriers with carrier -- FK \u2192 PK shorthand\njoin_one: origin_airport is airports on origin_airport.code = origin -- explicit ON\njoin_many: line_items on line_items.order_id = id -- one-to-many\njoin_cross: other_table on other_table.key = key -- cross join\n```\n\n- `join_one` \u2014 the joined source has at most one row per source row (many-to-one or one-to-one)\n- `join_many` \u2014 the joined source has potentially many rows per source row\n- `join_cross` \u2014 a full cross product\n\nThe `with` shorthand requires the joined source to have a declared `primary_key`. All joins are left outer by default. There is no right join \u2014 Malloy\'s graph model doesn\'t need one.\n\n**Choosing `join_one` vs `join_many`:** Ask "for a single row in the base source, can the joined source match more than one row?" If yes \u2192 `join_many`. If no (or at most one) \u2192 `join_one`. The common mistake is reaching for `join_many` when joining a *lookup or summary table* (e.g., joining an inventory snapshot to a purchase history on a wine key). Even though the joined table may have many rows overall, if each base row resolves to *at most one* joined row, use `join_one`. Use `join_many` only when the join genuinely fans out the base rows \u2014 e.g., joining line items to orders, or notes to a wine catalog.\n\nWhen you reference a joined source\'s fields, you use dot notation: `carriers.nickname`, `origin_airport.state`. This is one of Malloy\'s most important abstractions: **the access path to nested data is identical regardless of how the nesting is physically stored.** An array of records embedded in a column, a `join_many` to a separate table, a record-typed column \u2014 all are navigated with the same dot notation. The SQL required to traverse these different physical arrangements varies wildly (unnesting arrays, LEFT JOINs, correlated subqueries, ARRAY_AGG), but Malloy hides all of that. You think about the logical shape of your data \u2014 "flights have carriers, carriers have a nickname" \u2014 and write `carriers.nickname`. The compiler figures out what SQL is needed to get there. This means you can restructure your physical schema (normalize a nested array into a separate table, or denormalize a joined table into a record column) without changing any of the Malloy that references that data.\n\n## Fields\n\nMalloy has four kinds of fields: **dimensions**, **measures**, **views**, and **calculations**.\n\n### Dimensions\n\nScalar expressions \u2014 they compute a value per row. All columns inherited from a table are dimensions. Computed dimensions reference other dimensions or columns:\n\n```malloy\ndimension: full_name is concat(first_name, \' \', last_name)\ndimension: is_long_haul is distance > 1000\n```\n\n### Measures\n\nAggregate expressions \u2014 they compute a value across a set of rows. A field is a measure when its defining expression contains an aggregate function (`count`, `sum`, `avg`, `min`, `max`):\n\n```malloy\nmeasure:\n flight_count is count()\n total_distance is sum(distance)\n avg_distance is avg(distance)\n pct_delayed is count() { where: dep_delay > 30 } / count()\n```\n\n**`count(expr)` counts distinct values.** Unlike SQL\'s `COUNT(DISTINCT expr)`, Malloy uses `count(expr)` for distinct counting. The `count(distinct expr)` form is a deprecated syntax that will produce an error. Use `count()` for total row count, `count(field)` for distinct values of that field:\n\n```malloy\naggregate:\n total_rows is count() -- all rows\n unique_carriers is count(carrier) -- distinct carriers\n```\n\nMeasures can be filtered inline with `{ where: ... }`, which is how you build things like "percent of flights delayed" without subqueries.\n\n### Views\n\nA view is a query saved into the source \u2014 a reusable transformation:\n\n```malloy\nview: by_carrier is {\n group_by: carrier\n aggregate: flight_count, total_distance\n limit: 10\n}\n```\n\nViews can reference other views from the same source as a starting point, and can be extended with `+`.\n\n### Calculations\n\nWindow functions over the grouped result. Calculations can only be defined in a query stage with `calculate:`, never in a source definition, because they depend on the output columns of the query:\n\n```malloy\nrun: flights -> {\n group_by: carrier\n aggregate: flight_count\n calculate: carrier_rank is rank()\n}\n```\n\n## Queries and Views\n\nA query pairs a source with a view (the transformation). Everything after the first `->` is the view.\n\n```malloy\nrun: flights -> {\n group_by: carrier\n aggregate: flight_count\n}\n```\n\n### Reduction vs. Projection\n\nEach stage of a view performs exactly one of:\n\n- **Reduction** \u2014 uses `group_by:` and/or `aggregate:` to reduce grain. Analogous to `SELECT ... GROUP BY` in SQL.\n- **Projection** \u2014 uses `select:` to pick fields without aggregation. Analogous to `SELECT` without `GROUP BY`.\n\nThese cannot be mixed in a single stage. A stage with `group_by:` cannot have `select:`, and vice versa.\n\n### Source-level definitions vs. query-level operations\n\nThe same `name is expression` syntax defines fields in both sources and queries:\n\n```malloy\n-- In a source (reusable):\nsource: flights is ... extend {\n measure: flight_count is count() -- defines a measure in the model\n}\n\n-- In a query (ad hoc):\nrun: flights -> {\n aggregate: flight_count is count() -- defines the same measure inline\n}\n```\n\nWhen used in a source, `measure:` and `dimension:` are **definition statements** \u2014 they add named fields to the source\'s schema. When used in a query, `group_by:`, `aggregate:`, `select:`, `nest:`, and `calculate:` are **query operations** \u2014 they specify what the query does. The field definitions are syntactically identical in both contexts, but the enclosing keyword determines the role:\n\n| Source keyword | Query keyword | What it holds |\n|---|---|---|\n| `dimension:` | `group_by:` or `select:` | scalar expressions |\n| `measure:` | `aggregate:` | aggregate expressions |\n| `view:` | `nest:` | sub-queries |\n| _(n/a)_ | `calculate:` | window functions |\n\nThis is why `measure` and `aggregate` are separate keywords. `measure:` is a *modeling* statement \u2014 "this source has a reusable aggregate computation called X." `aggregate:` is a *query* statement \u2014 "in this query, include these aggregate values in the output." A query\'s `aggregate:` can reference a previously defined measure by name, or define one inline. The distinction parallels the separation between defining a dimension in a source and using it via `group_by:` in a query.\n\n### Multi-stage Pipelines\n\nStages chain with `->`. Each stage\'s output becomes the next stage\'s source:\n\n```malloy\nrun: flights -> {\n group_by: carrier\n aggregate: flight_count is count()\n} -> {\n where: flight_count > 1000\n select: *\n}\n```\n\n### Refinement with `+`\n\nThe refinement operator `+` merges query operations together. It works both within a view and at the top level on a named query:\n\n```malloy\n-- Refining a view within a query:\nrun: flights -> by_carrier + { limit: 5 } + { nest: by_destination }\n\n-- Refining a named query at the top level:\nrun: carrier_summary + { group_by: origin } -- add origin grouping to existing query\n```\n\nWhen a dimension name appears as a bare reference, it expands to `{ group_by: name }`. A measure name expands to `{ aggregate: name }`:\n\n```malloy\nrun: flights -> carrier + flight_count + { limit: 10 }\n-- equivalent to: flights -> { group_by: carrier; aggregate: flight_count; limit: 10 }\n```\n\nFor multi-stage queries, refinement semantics get more complex \u2014 but for single-stage queries, `+` straightforwardly merges operations into the stage.\n\n### Nesting\n\n`nest:` embeds an aggregating subquery inside a reduction. Each row of the outer query gets a subtable from the nested query. Nests can nest arbitrarily deep:\n\n```malloy\nrun: flights -> {\n group_by: carrier\n aggregate: flight_count\n nest: top_routes is {\n group_by: origin, destination\n aggregate: flight_count\n limit: 3\n }\n}\n```\n\n### Other query operations\n\n- **`where:`** \u2014 filter rows (pre-aggregation). Comma-separated filters are ANDed.\n- **`having:`** \u2014 filter groups (post-aggregation), like SQL\'s HAVING.\n- **`limit:`** / **`order_by:`** \u2014 limit and sort output.\n- **`extend`** \u2014 add fields or joins to a source inline within a query expression.\n\n## Aggregate Locality (Symmetric Aggregates)\n\nThis is one of Malloy\'s most important features. In SQL, when you join tables and aggregate, you risk double-counting (the "fan trap"). Malloy solves this with **aggregate locality** \u2014 you specify *where in the join graph* an aggregation should be computed.\n\n```malloy\nrun: flights -> {\n aggregate:\n -- avg seats weighted by number of flights (locality: source, i.e. flights)\n avg_seats_per_flight is source.avg(aircraft.aircraft_models.seats)\n -- avg seats per aircraft model (locality: aircraft_models)\n avg_seats_per_model is aircraft.aircraft_models.seats.avg()\n}\n```\n\nThree syntactic forms:\n\n- `avg(expr)` \u2014 aggregate at the current source (implicit locality)\n- `joined_source.avg(expr)` \u2014 aggregate at the specified join point (explicit locality)\n- `joined_source.field.avg()` \u2014 shorthand for aggregating the field at its parent source\n\nFor `sum` and `avg` (asymmetric aggregates), when the expression crosses a join boundary, Malloy *requires* explicit locality \u2014 it won\'t silently give you a wrong answer. For `min`, `max`, and `count` (symmetric), locality doesn\'t change the result, so implicit is always fine.\n\nMalloy implements this with a technique called **symmetric aggregates** \u2014 it internally de-duplicates rows based on primary keys at the appropriate join level, so aggregations are always mathematically correct regardless of join fan-out.\n\n## Ungrouped Aggregates\n\n`all()` and `exclude()` allow computing aggregates at different grouping levels within a single query:\n\n```malloy\nrun: airports -> {\n group_by: state, faa_region\n aggregate:\n airport_count is count()\n total_airports is all(count()) -- ungrouped: total across all rows\n region_airports is all(count(), faa_region) -- grouped only by faa_region\n pct_of_total is count() / all(count())\n}\n```\n\n`all(expr)` removes all grouping. `all(expr, dim1, dim2)` keeps only the specified grouping dimensions. `exclude(expr, dim)` removes the specified dimension from grouping.\n\n**Important:** `all(expr, dim)` takes the **local alias name** as defined in the query\'s `group_by:`, not a dotted path. If you want to partition by a joined field, alias it first:\n\n```malloy\n-- WRONG: all(count(), director.primaryName) -- dot paths don\'t work here\n-- RIGHT:\nrun: movies -> {\n group_by: director is director.primaryName -- alias it\n aggregate:\n movies is count()\n director_total is all(count(), director) -- reference the alias\n pct is count() / all(count(), director)\n}\n```\n\n## Expressions\n\nMalloy expressions include arithmetic, comparison, logical operators, function calls, type casts, and several Malloy-specific forms.\n\n### Evaluation Spaces\n\nEvery expression has an evaluation space: **literal**, **constant**, **input**, or **output**. Input expressions reference source columns/dimensions. Output expressions reference the results of the current query stage (used in `calculate:`). Some functions constrain their arguments \u2014 e.g., `lag(expr)` requires an output expression, `avg(expr)` requires an input expression.\n\n### Application and Partial Comparison\n\nThe `?` operator applies a condition to a value. Partial comparisons are conditions without a left-hand side:\n\n```malloy\nwhere: state ? \'CA\' | \'NY\' -- state is \'CA\' or \'NY\'\nwhere: distance ? > 500 & < 2000 -- distance between 500 and 2000\n```\n\n`|` is alternation (OR), `&` is conjunction (AND) within partials.\n\n### Pick Expressions\n\nMalloy\'s equivalent of CASE:\n\n```malloy\ndimension: size_bucket is\n pick \'short\' when distance < 500\n pick \'medium\' when distance < 1500\n else \'long\'\n```\n\n### Filtered Aggregate Expressions\n\nAny aggregate can be filtered inline:\n\n```malloy\nmeasure: ca_flights is count() { where: origin.state = \'CA\' }\n```\n\n### Type Casting\n\n```malloy\ntotal_distance::string -- Malloy type cast\nname::"VARCHAR(32)" -- database-native type cast\n```\n\n### Time Literals and Ranges\n\n```malloy\n@2003 -- the year 2003\n@2003-Q2 -- second quarter of 2003\n@2024-01-15 10:30:00 -- timestamp literal\ndep_time ? @2003 to @2005 -- range comparison\nnow -- current timestamp\n```\n\n## Data Types\n\nMalloy\'s type system: `string`, `number`, `boolean`, `date`, `timestamp`, `timestamptz`, `json`, and `sql native` (for unsupported database types). Compound types: `type[]` for arrays, `{ name :: type, ... }` for records, nesting arbitrarily: `{ x :: number, tags :: string[] }[]`.\n\n## Annotations and Tags\n\nThese are related but distinct concepts.\n\n### Annotations\n\nAnnotations are **text strings** attached to objects during compilation. They are metadata \u2014 they never affect query execution or SQL generation. An annotation starts with `#` and continues to end of line:\n\n```malloy\n# bar_chart\nview: by_carrier is { ... }\n```\n\n- `#` annotations attach to the next object defined below them\n- `##` annotations attach to the model (the file)\n- Block annotations use `#|` ... `|#` for multi-line content (closing delimiter must match the column position of the opener)\n\nAnnotations distribute over definition lists:\n\n```malloy\n# currency\nmeasure: -- all three measures get the # currency annotation\n revenue is sum(amount)\n # percent -- this measure also gets # percent\n margin is revenue / cost\n cost is sum(amount)\n```\n\n### Tags (a use of annotations)\n\nTags are the primary *consumer* of annotation strings. They interpret annotation text using a structured property language (MOTLY). The key distinction: **annotations are the transport mechanism (raw strings attached to objects), tags are the interpretation layer (parsed key-value properties).**\n\nNot all annotations are tags. An annotation is just text. Tags are annotations that happen to be written in the tag property language and parsed by an application.\n\n### Annotation prefixes (routing)\n\nThe character(s) immediately after `#` route the annotation to different consumers:\n\n- `# ` (hash-space) \u2014 renderer tags, parsed by the Malloy VS Code extension for visualization (`# bar_chart`, `# line_chart`, `# shape_map`; `# link { url_template="https://\u2026/$$" }` turns a field\'s cell into a clickable deep link \u2014 `$$` is the cell value, `field=other` links on a different (often `# hidden`) field; `# image { url_template=\u2026 }` renders it as an image)\n- `##!` \u2014 compiler directives (e.g., `##! experimental.parameters`, `##! experimental.givens`)\n- `#"` \u2014 reserved for documentation strings\n- `#(appName)` \u2014 application-specific tags (e.g., `#(docs) hidden`)\n\n```malloy\n# bar_chart size=large -- renderer tag: tells VS Code how to render\n##! experimental.parameters -- compiler tag: enables a feature flag\n#(myApp) priority=high -- custom app tag: ignored by renderer/compiler\n```\n\n### Tag property syntax\n\n```\ntName -- boolean flag (exists = true)\ntName=value -- set property value\ntName=[a, b, c] -- array value\ntName: { p1=v1 p2=v2 } -- nested properties (replaces)\ntName { p1=v1 } -- nested properties (merges)\n-tName -- delete a property\ntName.sub.path=value -- deep path assignment\n```\n\nValues can be unquoted identifiers, quoted strings, numbers, or typed values prefixed with `@` (`@true`, `@false`, `@2024-01-15`).\n\n## Givens (Model-Level Parameters)\n\n**Status: experimental, gated by `##! experimental.givens`.** Naming is provisional.\n\nGivens are values supplied at run time that the model can reference in any expression. The motivating use case is row-level access control \u2014 a model written once with `where: x.tenant_id = $TENANT` and the tenant supplied per API call \u2014 but they also fit configuration values, session context, and any "one compiled model, many invocations with varying context" pattern.\n\nGivens are model-wide: a single namespace, one value per name per compilation. They are *complementary to* source/query parameters (`source: foo(x :: string) is ...`), not a replacement. Use a parameter when you want two differently-bound copies of the same source side-by-side in one model; use a given when you want one value visible everywhere in the compilation.\n\n### Declaration\n\nThe `given:` top-level statement introduces givens, with a name, a type, and an optional default:\n\n```malloy\ngiven:\n TENANT :: string\n MAX_ROWS :: number is 1000\n CUTOFF_DATE :: date is @2024-01-01\n```\n\nType can be any Malloy atomic type or compound type, including `filter<T>`:\n\n```malloy\ngiven:\n ROLE :: string\n ALLOWED_ROLES :: string[]\n SESSION :: { user_id :: string, tenant :: string }\n TENANT_FILTER :: filter<string>\n```\n\nDefaults are expressions over constants and other givens. Annotations attach to given declarations the same way they attach to sources or measures.\n\n### Reference: the `$` sigil\n\nInside any expression, a given is referenced with a leading `$`:\n\n```malloy\nsource: orders_for_user is orders extend {\n where: orders.tenant_id = $TENANT\n}\n\nquery: recent_orders is orders_for_user -> {\n where: order_date >= $CUTOFF_DATE\n limit: $MAX_ROWS\n}\n```\n\n`$` appears *only* at expression references. The other three sites where a given\'s name appears \u2014 declaration, import, and supply (caller side) \u2014 use the bare name, because syntactic position already disambiguates. Givens share the top-level declaration namespace with sources/queries/views, so `source: x is ...` plus `given: x :: string` is a name-conflict error.\n\n### Set membership: `expr in $arrayGiven`\n\nThe RHS of `in` is either a parenthesized list of expressions (`in (1, 2, x, y * 7)`, same as SQL) or a given with an array value (`in $ARR`). A bare array-typed expression \u2014 a dimension, a joined array field, an inline `[a, b, c]` literal \u2014 is *not* legal on the RHS; arrays only reach the RHS via the given form.\n\nWhen a given has array type, `expr in $ARR` tests `expr` against the runtime-bound array; `not in $ARR` is the negation. The left-hand side must match the array\'s element type (`string in $string[]`, `number in $number[]`, etc.); mismatches are translate-time errors. Records and nested arrays are out of scope.\n\n```malloy\ngiven:\n ALLOWED_STATES :: string[]\n URGENT_STATUSES :: string[]\n\nsource: orders extend {\n where: state in $ALLOWED_STATES\n dimension: is_urgent is order_status in $URGENT_STATUSES\n}\n```\n\nAt SQL emit, the array\'s contents land in a generated `IN (...)` clause. Empty or `null` arrays collapse to the obvious result (`IN` \u2192 `FALSE`, `NOT IN` \u2192 `TRUE`). NULL elements inside a non-empty array follow standard SQL `IN` semantics.\n\nTo derive a value from an array \u2014 typically a boolean gate \u2014 *without* the array itself reaching row-position SQL, use an inline given (below).\n\n### Inline givens\n\nAn `inline` given is evaluated at **bind time**, before SQL is emitted: its default expression runs against the resolved given values and reduces to a literal, and that literal is what reaches SQL.\n\n```malloy\ngiven:\n CAPABILITIES :: string[]\n inline CAN_READ_ORDERS :: boolean is \'read_orders\' in $CAPABILITIES\n inline CAN_MUTATE :: boolean\n is \'write_orders\' in $CAPABILITIES or \'admin\' in $CAPABILITIES\n\nsource: orders extend {\n where: $CAN_READ_ORDERS -- SQL sees: WHERE ... AND TRUE (or FALSE)\n}\n```\n\nThis is the **row-level access-control gate** pattern: the host supplies a capability list as a regular given, an inline given derives a boolean from it, and only the boolean \u2014 not the list \u2014 crosses into row-position SQL. The query planner sees a constant predicate.\n\nRules:\n\n- An inline given **must** have a default. `inline FOO :: number` with no `is` clause is a translate-time error.\n- The default may use:\n - Boolean and comparison operators: `and`, `or`, `not`, `=`, `!=`, `<`, `<=`, `>`, `>=`\n - The `in $array` test against another given\n - Literals (string, number, boolean, null, array) and references to other givens\n- The default cannot call SQL functions, reference fields, or use any operator outside that list. Disallowed operators are reported at translate time with the offending operator names.\n- Inline givens are filtered out of `Model.givens` and `PreparedQuery.givens` \u2014 they\'re computed, not supplied \u2014 so introspection-driven UIs don\'t render editors for them. A caller can still shadow one by binding it explicitly (useful in tests).\n- `inline` is a context-sensitive modifier, not a reserved keyword: fields, sources, views, dimensions, and joins can still be named `inline`.\n\n### Imports\n\nGivens behave like every other top-level named thing under import:\n\n- **Bare import** (`import "b.malloy"`) brings B\'s full export surface in, including all of B\'s givens, under their original names.\n- **Selective import** (`import { source1 } from "b.malloy"`) brings in only what\'s listed. To surface a given to your callers, list it: `import { source1, MAX_ROWS } from "b.malloy"`.\n- **Rename** uses the existing `LOCAL is REMOTE` form: `import { CAP is MAX_ROWS } from "b.malloy"`.\n\nSurfacing controls *who can supply a value*, not whether internal references work. An imported source can reference a given the importer didn\'t surface; the reference still resolves internally, and at run time the unsurfaced given relies on its declaration-site default.\n\nA common project convention is a shared `tenant_givens.malloy` (declaring `$TENANT`, `$USER_ROLE`, etc.) that every root file bare-imports on line 1, so the project\'s given contract is visible at the top of any model.\n\n### Satisfiability\n\nA query referencing `$X` is satisfiable if either `$X` is in the model\'s namespace (so a caller can supply a value) or `$X` has a default at its declaration site. Otherwise the query is unsatisfiable and errors. Latent definitions (views, dimensions, measures) that reference `$X` are fine if no query actually invokes them \u2014 satisfiability is a property of running queries.\n\n### Supplying values\n\nValues can be supplied at two layers, which compose (per-query overrides per-runtime):\n\n**Per-runtime** \u2014 bound to a `Runtime`, applied as defaults to every query through it. Two paths:\n\n1. **`givensPath` in `malloy-config.json`** points at a JSON file of `name \u2192 value`:\n ```jsonc\n { "givensPath": "./local-givens.json" }\n // or env-var indirection (resolved at config load):\n { "givensPath": { "env": "GAME_STORE_GIVENS" } }\n ```\n The values file is a flat JSON map, keys are caller-facing surface names:\n ```jsonc\n { "TENANT": "acme", "USER_ROLE": "admin", "CUTOFF_DATE": "2024-01-01" }\n ```\n\n2. **Direct on the Runtime constructor** (for per-request multi-tenant servers, tests, scripts):\n ```typescript\n const runtime = new Runtime({\n config,\n givens: { TENANT: claims.tenant_id, USER_ROLE: claims.role },\n urlReader,\n });\n ```\n Constructor values *merge over* the file at `givensPath` per-key.\n\n**Per-query** \u2014 supplied on a single `.run({ givens: ... })` call:\n```typescript\nawait query.run({ givens: { STATE_FILTER: "CA", LIMIT_OVERRIDE: 50 } })\n```\nAvailable on every compile-or-run entry point (`runtime.loadQuery(...).run(options)`, `preparedQuery.getPreparedResult(options)`, `preparedQuery.getSQL(options)`).\n\nThe resolved per-runtime values are exposed on `runtime.givens` (read-only) for diagnostics.\n\n### Finalized givens (security primitive)\n\nA multi-tenant deployment usually wants `TENANT`/`USER_ROLE`/`REGION` to be runtime-bound and **un-overridable per-query** \u2014 otherwise a downstream endpoint that accidentally accepts user-controlled query params and plumbs them into `.run({ givens: ... })` becomes a tenant-leak vulnerability.\n\n`finalizeGivens` in the config locks names at the API surface:\n\n```jsonc\n{\n "givensPath": { "env": "GAME_STORE_GIVENS" },\n "finalizeGivens": ["TENANT", "USER_ROLE", "REGION"]\n}\n```\n\nFinalize doesn\'t change *what* a given resolves to \u2014 only *who* can supply it. A `.run({ givens: { TENANT: ... } })` for a finalized name throws at API entry (named, not silently dropped). Finalized givens are filtered out of `Model.givens` and `PreparedQuery.givens` so introspection-driven UIs don\'t render editors for locked names.\n\n### JS shapes for supplied values\n\nBoth the JSON values file and per-query `givens` maps accept the same per-type shapes:\n\n| Malloy type | JS |\n|---|---|\n| `string` | string |\n| `number` | `number`, `bigint`, or string (precision escape hatch) |\n| `boolean` | boolean |\n| `date` | ISO date string `"2024-01-15"` |\n| `timestamp` (naive) | ISO string without offset \u2014 *not* a JS `Date` |\n| `timestamptz` | JS `Date` or ISO string with offset (string preferred \u2014 makes TZ choice visible) |\n| `T[]` | JS array |\n| `{ name :: T, ... }` | JS object |\n| `filter<T>` | JS string (Malloy filter expression source) |\n\nNaive timestamp givens reject `Date` because `Date` represents a UTC instant, not a wall-clock value, and `new Date("2001-01-01T00:00:00")` silently picks up the system\'s local TZ. Type mismatches throw at the boundary with a path that points at the offending location (e.g., `givens.SESSION.user_id: expected string, got number`). `null` is legal for any given type.\n\n### Introspection\n\n`Model.givens` and `PreparedQuery.givens` expose, to the host, the supplyable givens \u2014 for whole-model parameter editors and per-query "run this" forms respectively. Each entry carries name, type, default expression (or undefined if the caller must supply), location, and access to declaration-site annotations via `tagParse`/`getTaglines`.\n\n## How a Malloy Query Becomes SQL\n\nThe compilation pipeline has two phases:\n\n### Phase 1: Translation (source code \u2192 IR)\n\n```\nMalloy source \u2192 ANTLR lexer/parser \u2192 parse tree \u2192 AST builder \u2192 AST \u2192 IR generator \u2192 IR\n```\n\nThe **Intermediate Representation (IR)** is a plain, serializable data structure (JSON-compatible) that fully describes the semantic model and query. Note that IR is *not* dialect-agnostic \u2014 the same Malloy source compiled against different databases can produce different IR, because schema information, type mappings, and available functions vary by backend. It can be cached, transmitted, and reused. Key IR types:\n\n- **`SourceDef`** \u2014 a source\'s complete definition: schema, fields, joins, filters\n- **`Query`** \u2014 a source paired with a pipeline of operations\n- **`FieldDef`** \u2014 definition of any field (dimension, measure, join, calculation)\n- **`Expr`** \u2014 expression tree (arithmetic, comparisons, aggregates, function calls, field references)\n\nThe translator handles all language-level semantics: scoping, name resolution, type checking, evaluation space validation.\n\n### Phase 2: Compilation (IR \u2192 SQL)\n\n```\nIR \u2192 query compiler \u2192 expression compiler \u2192 dialect-specific SQL generator \u2192 SQL + metadata\n```\n\nThe compiler walks the IR query pipeline, translating each stage into SQL constructs (CTEs, subqueries, GROUP BY, window functions). A **Dialect** layer handles database-specific SQL generation.\n\nThe compiler also produces **metadata** alongside the SQL \u2014 structural information needed to interpret the result set (column types, nesting structure, annotation data). This metadata is what allows Malloy renderers to reconstruct nested/hierarchical results from the flat SQL result set and apply visualization tags.\n\n### Key architectural consequences\n\n- Because the IR is serializable, it can be cached and reused across compilations (though IR is database-specific \u2014 the same source compiled against different backends may produce different IR).\n- Because joins are declared in the source (not the query), the compiler knows the full join graph and can compute symmetric aggregates correctly.\n- Because nested queries are first-class, the compiler generates the appropriate SQL (correlated subqueries or ARRAY_AGG patterns depending on dialect) automatically.\n- Because measures are typed as aggregates in the IR, the compiler can validate that they only appear in aggregate context and enforce locality rules.\n\n## Where to Go Deeper\n\nThis document is a conceptual reference \u2014 enough to reason about the language and its design, but not exhaustive. Here\'s where to find more detail.\n\n### Language Documentation\n\nThe full docs live at [https://docs.malloydata.dev](https://docs.malloydata.dev). Key pages by topic:\n\n| Topic | URL |\n|---|---|\n| Sources, extensions, joins, primary keys | [documentation/language/source](https://docs.malloydata.dev/documentation/language/source) |\n| Queries, views, reduction vs projection | [documentation/language/query](https://docs.malloydata.dev/documentation/language/query), [views](https://docs.malloydata.dev/documentation/language/views) |\n| Fields: dimensions, measures, views, calculations | [documentation/language/fields](https://docs.malloydata.dev/documentation/language/fields) |\n| Aggregate functions and aggregate locality | [documentation/language/aggregates](https://docs.malloydata.dev/documentation/language/aggregates) |\n| Ungrouped aggregates (`all`, `exclude`) | [documentation/language/ungrouped-aggregates](https://docs.malloydata.dev/documentation/language/ungrouped-aggregates) |\n| Nested views / aggregating subqueries | [documentation/language/nesting](https://docs.malloydata.dev/documentation/language/nesting) |\n| Joins | [documentation/language/join](https://docs.malloydata.dev/documentation/language/join) |\n| Expressions, operators, pick, apply | [documentation/language/expressions](https://docs.malloydata.dev/documentation/language/expressions) |\n| Evaluation spaces (literal, constant, input, output) | [documentation/language/eval_space](https://docs.malloydata.dev/documentation/language/eval_space) |\n| Filters and filter placement | [documentation/language/filters](https://docs.malloydata.dev/documentation/language/filters) |\n| Annotations and tags | [documentation/language/tags](https://docs.malloydata.dev/documentation/language/tags) |\n| Calculations and window functions | [documentation/language/calculations_windows](https://docs.malloydata.dev/documentation/language/calculations_windows) |\n| Data types | [documentation/language/datatypes](https://docs.malloydata.dev/documentation/language/datatypes) |\n| Time operations, ranges, timezones | [documentation/language/timestamp-operations](https://docs.malloydata.dev/documentation/language/timestamp-operations), [time-ranges](https://docs.malloydata.dev/documentation/language/time-ranges), [timezones](https://docs.malloydata.dev/documentation/language/timezones) |\n| Imports | [documentation/language/imports](https://docs.malloydata.dev/documentation/language/imports) |\n| Top-level statements and model structure | [documentation/language/statement](https://docs.malloydata.dev/documentation/language/statement) |\n| Functions reference | [documentation/language/functions](https://docs.malloydata.dev/documentation/language/functions) |\n\n### Examples and Patterns\n\nThe docs site includes worked examples of common analytical patterns at [documentation/patterns](https://docs.malloydata.dev/documentation/patterns/): percent-of-total, year-over-year, cohort analysis, sessionization, moving averages, nested subtotals, and more.\n\nEnd-to-end guides are at [documentation/user_guides](https://docs.malloydata.dev/documentation/user_guides/), including [Malloy by Example](https://docs.malloydata.dev/documentation/user_guides/malloy_by_example) (a comprehensive walkthrough) and a three-part series for SQL users ([part 1](https://docs.malloydata.dev/documentation/user_guides/sql_experts1), [part 2](https://docs.malloydata.dev/documentation/user_guides/sql_experts2), [part 3](https://docs.malloydata.dev/documentation/user_guides/sql_experts3)).\n\n### Source Code\n\nThe Malloy implementation lives at [github.com/malloydata/malloy](https://github.com/malloydata/malloy). Key entry points:\n\n| What | Where |\n|---|---|\n| ANTLR grammar (lexer + parser) | `packages/malloy/src/lang/grammar/` |\n| AST node hierarchy | `packages/malloy/src/lang/ast/` |\n| Parse tree \u2192 AST builder | `packages/malloy/src/lang/malloy-to-ast.ts` |\n| IR type definitions | `packages/malloy/src/model/malloy_types.ts` |\n| IR \u2192 SQL compiler | `packages/malloy/src/model/` |\n| Dialect-specific SQL generation | `packages/malloy/src/dialect/` |\n| Tag/annotation parsing (MOTLY) | `packages/malloy-tag/` |\n| Renderer | `packages/malloy-render/` |\n| Architecture overview | `CONTEXT.md` (root and in each package) |\n',
|
|
217
212
|
"language/pick.md": "---\ndescription: pick expressions \u2014 Malloy's CASE/if-then-else\n---\n\n`pick` is Malloy's equivalent of SQL `CASE WHEN`. Each branch is its own\n`pick` keyword; the `else` clause catches the remainder.\n\nThere are two forms of pick. In the first the `when` expression is any\nboolean expression.\n\n```malloy\n pick 'Female' when upper(first_name) in ('JENNIFER', 'ELIZABETH', 'AMY', 'JESSICA')\n pick 'Male' when upper(first_name) in ('JAMES', 'JOHN', 'ROBERT', 'MICHAEL')\n else 'Unknown'\n```\n\n## Example usage in a query\n\n```malloy\nrun: payments -> {\n group_by: tier is\n pick 'high' when total_amount > 10000\n pick 'medium' when total_amount > 1000\n else 'low'\n aggregate: payment_count is count()\n}\n```\n\n## Common mistakes\n\n- **Every branch needs its own `pick` keyword** \u2014 there is no `when \u2026 then`:\n ```malloy\n -- WRONG:\n pick 'a' when x = 1 'b' when x = 2 else 'c'\n\n -- RIGHT:\n pick 'a' when x = 1\n pick 'b' when x = 2\n else 'c'\n ```\n\n- **`else` is required** when the branches don't cover all cases \u2014 omitting it\n returns `null` for unmatched rows.\n",
|
|
218
|
-
"language/who-is-asking.md": "---\ndescription: Per-user data with $MALLOYYO_EMAIL \u2014 the reserved given this server fills from the signed-in user, how to declare it, and how to run the model locally\n---\n\n# Who is asking \u2014 `$MALLOYYO_EMAIL`\n\nA model can scope what a query returns to the person running it. Declare the\nreserved given, then use it like any other:\n\n```malloy\n##! experimental.givens\ngiven:\n MALLOYYO_EMAIL :: string is ''\n\nsource: orders is duckdb.table('orders.parquet') extend {\n where: owner_email = $MALLOYYO_EMAIL\n}\n```\n\nEvery query against that model \u2014 the `query` tool, a dashboard tile, a saved\nquery, an ltool link \u2014 binds it to the signed-in user's address. Nothing else\nchanges: no per-user models, no filtered copies, no query rewriting.\n\n**The declaration is the opt-in.** A model that does not declare it is\nuntouched, because supplying a given a model never declared is an error in\nMalloy rather than a no-op.\n\n## What a caller cannot do\n\nA caller may not set it. A `givens: { \"MALLOYYO_EMAIL\": \"someone@else.com\" }`\non a `query` call, a dashboard's `?MALLOYYO_EMAIL=` URL, a value baked into a\ncomponent \u2014 all dropped before the query compiles, and the signed-in address\nbound instead. That is the whole point of the feature; do not design around it.\n\nFor the same reason it is not offered as a filter: `query(execute:false)` does\nnot list it among the givens to supply, and a dashboard renders no control for\nit.\n\n## The `MALLOYYO_` prefix is reserved\n\
|
|
213
|
+
"language/who-is-asking.md": "---\ndescription: Per-user data with $MALLOYYO_EMAIL \u2014 the reserved given this server fills from the signed-in user, how to declare it, and how to run the model locally\n---\n\n# Who is asking \u2014 `$MALLOYYO_EMAIL`\n\nA model can scope what a query returns to the person running it. Declare the\nreserved given, then use it like any other:\n\n```malloy\n##! experimental.givens\ngiven:\n MALLOYYO_EMAIL :: string is ''\n\nsource: orders is duckdb.table('orders.parquet') extend {\n where: owner_email = $MALLOYYO_EMAIL\n}\n```\n\nEvery query against that model \u2014 the `query` tool, a dashboard tile, a saved\nquery, an ltool link \u2014 binds it to the signed-in user's address. Nothing else\nchanges: no per-user models, no filtered copies, no query rewriting.\n\n**The declaration is the opt-in.** A model that does not declare it is\nuntouched, because supplying a given a model never declared is an error in\nMalloy rather than a no-op.\n\n## What a caller cannot do\n\nA caller may not set it. A `givens: { \"MALLOYYO_EMAIL\": \"someone@else.com\" }`\non a `query` call, a dashboard's `?MALLOYYO_EMAIL=` URL, a value baked into a\ncomponent \u2014 all dropped before the query compiles, and the signed-in address\nbound instead. That is the whole point of the feature; do not design around it.\n\nFor the same reason it is not offered as a filter: `query(execute:false)` does\nnot list it among the givens to supply, and a dashboard renders no control for\nit.\n\n## The `MALLOYYO_` prefix is reserved\n\nTwo exist: `MALLOYYO_EMAIL` (the signed-in address) and `MALLOYYO_ROLES` (every\nrole that person holds, as a list \u2014 `where: department in $MALLOYYO_ROLES`).\nDeclaring any other `MALLOYYO_*` given is refused at publish: the prefix means\n\"a value this server vouches for\", and a `$MALLOYYO_ROLE` nothing fills would\nread as though it did. Name your own givens anything else.\n\n## The dataset decides whether it is supplied\n\nDeclaring the given is not enough. An admin ticks what a dataset is scoped by,\nand only then is the value supplied and locked. The two halves check each\nother:\n\n- tick it, and a model that does not declare it **cannot be published** \u2014 the\n data would be served unscoped;\n- declare it without the tick, and nothing supplies it, so your filter runs on\n the declaration default. `publish` says so in that case; do not ignore the\n line.\n\nThat is why the default matters so much (see Notes below): an unsupplied\n`:: string is ''` matches nothing, which is the safe way to be misconfigured.\n\n## Running it locally\n\nLocally there is no one asking, so a tenant-scoped query would come back\nempty \u2014 which looks exactly like a broken dashboard. Set the address in the\nenvironment instead:\n\n```bash\nMALLOYYO_EMAIL=you@example.com malloyyo dashboard dev\n```\n\n`malloyyo lint` needs nothing: it compiles, it does not run. Anything that\nactually runs a query refuses without a value and names the variable, rather\nthan showing you an empty page to debug. Change the address to see another\ntenant's view.\n\nNothing about the identity is committed. On a server the value comes from\nwhoever is signed in; the environment is only the local stand-in, because a\nserver's environment is ONE value for every request and would hand every user\nthe same tenant.\n\n## Not `malloyyo dashboard bundle`\n\nA bundled site has no one asking, and cannot have: there is no server, the\nmodel source is inlined into the page, and DuckDB-WASM fetches the data in the\nreader's own browser. Nothing supplies the identity there \u2014 the bundler only\nintrospects, and no environment travels into a static page. So a tenant-scoped\ndashboard bundles to a page that shows nothing.\n\nDo not answer that by loosening the default. `MALLOYYO_EMAIL :: filter<string>\nis f'\u2026'` with anything permissive publishes every row to every visitor, and the\nraw data is one fetch away regardless. **A model that filters by who is asking\nbelongs on an instance, not in a static bundle.**\n\n## Notes\n\n- Give it a default and make sure the default is the SAFE answer. Watch the\n type: `:: string is ''` matches no rows, which is right, but\n `:: filter<string> is f''` is an EMPTY FILTER, which matches EVERY row. The\n server refuses to run a query when it cannot supply the value (an account\n with no address, say) rather than let a default like that through \u2014 but the\n default is still what a local `dashboard dev` run and any non-server path\n will use, so write one that shows nothing.\n- Filtering belongs in the source (`extend { where: \u2026 }`), not in each query \u2014\n that way it holds for every query anyone writes against it, including ones\n an agent composes later.\n- It is a string, not a verified claim about identity beyond sign-in: it is the\n address the signed-in account carries.\n",
|
|
214
|
+
"repo/convert-single-dataset.md": "---\ndescription: Move an old single-dataset repo (index.malloy at the root) to the datasets/ layout\n---\n\n# Convert a single-dataset repo\n\nThe old layout put `index.malloy` at the repo root. To move it under `datasets/`:\n\n1. **Pick the dataset's name.** It becomes the directory name and the published\n name, so use what the dataset is already called on the instance \u2014 renaming it\n here would publish a second dataset rather than update the first.\n\n2. **Move everything the model owns**, keeping `malloy-config.json` at the root:\n\n mkdir -p datasets/<name>\n git mv index.malloy *.malloy dashboards datasets/<name>/\n\n Leave at the root: `malloy-config.json`, `.mcp.json`, `.devcontainer/`,\n `.claude/`, `README.md`, and anything that is not the model.\n\n3. **Fix imports that reached outside the model.** Imports between files that\n moved together still resolve \u2014 they kept their relative positions. Only a path\n that pointed at something left behind needs editing, and it needs `../../`\n in front of it.\n\n4. **Verify before publishing:**\n\n malloyyo lint\n\n Lint reads the repo the same way the server does, so a clean lint here means\n the server will accept it. It names the file and line for anything that moved\n wrong.\n\n5. **Publish the repo**, not the dataset:\n\n malloyyo publish --repo <owner/name>\n\n The dataset already exists on the instance, so no `--create-datasets`. The\n name matching is what connects the moved directory to the dataset that was\n already there \u2014 which is why step 1 matters.\n\n### If the dataset is backed by GitHub rather than pushed\n\nAn instance that pulls from the repo needs to learn the dataset moved. The repo's\nlayout is re-read on every refresh, so pushing the converted repo and refreshing\nis enough \u2014 but the dataset's recorded directory is set when it is created. If\nthe refresh reports the dataset as **unpublished** (the repo no longer has its\nold directory) and the new directory as **unclaimed**, an admin updates the\ndataset's directory on the instance, or re-adds the repo.\n\nNothing is deleted either way: a dataset whose directory disappears keeps\nserving its last model until someone decides otherwise.\n\n---\n\n*This topic is transitional. It exists while repos built on the old\nsingle-dataset layout are still in the wild, and goes away with them \u2014 along with\nthe notice `lint` and `publish` print when they meet one.*\n",
|
|
215
|
+
"repo/datasets.md": '---\ndescription: A model repo holds datasets under datasets/ \u2014 add one, or convert a single-dataset repo to the new layout\n---\n\n# Datasets in a model repo\n\nA model repo publishes **datasets**. Each one is a directory under `datasets/`,\nnamed after the directory:\n\n```\nmalloy-config.json connections \u2014 shared by every dataset\ndatasets/\n finance/\n index.malloy what THIS dataset publishes\n orders.malloy the model behind it\n dashboards/\n spend.malloy\n sales/\n malloy-config.json \u2026unless a dataset brings its own\n index.malloy\n dashboards/\nlib/ shared code, imported by relative path\n```\n\n`malloyyo init` scaffolds the repo and nothing else \u2014 no datasets. You add them.\n\n## The rules worth knowing before you start\n\n- **The directory names the dataset.** `datasets/finance/` publishes `finance`.\n The name is a slug \u2014 it is the URL, what a publish matches on, and what a role\n is granted against \u2014 so a directory that is not already one is converted:\n `datasets/the-look/` publishes `the_look`. `malloyyo lint` prints the name it\n will publish as, next to the directory.\n- **A dataset can say what to CALL it and what it IS**, at model scope in its\n `index.malloy`:\n\n ##" Deals, contacts and companies, as the sales team sees them.\n ## dataset { title="HubSpot CRM" }\n\n Note `##"` \u2014 two hashes. A `#"` attaches to whatever declaration follows it,\n so `#"` above the first source documents that SOURCE and leaves the dataset\n with no description. The description shows where someone is deciding whether\n they want this dataset, or who should see it.\n\n With no tag the title is derived from the name \u2014 `hub_spot` \u2192 "Hub Spot" \u2014\n so a dataset that says nothing still reads properly. Use the tag when the\n mechanical version is wrong: an acronym, a product\'s own capitalisation.\n The title is presentation ONLY. It is not unique, nothing is looked up by it,\n and changing it renames nothing.\n- **A dataset publishes exactly what its own `index.malloy` exports.** A source\n next door is not hidden from it, it is absent.\n- **`malloy-config.json` can be at the repo root or in a dataset.** At the root\n it is shared by every dataset. In a dataset it is that dataset\'s own \u2014 and it\n REPLACES the root\'s rather than adding to it, so a dataset that brings its own\n cannot reach a connection only the root declares. Nearest one wins.\n- **The repo publishes as a unit.** Every dataset is linted, and the server\n writes all of them or none.\n- **Never have both layouts.** A root `index.malloy` AND a `datasets/` directory\n is refused. A repo with a root `index.malloy` is the old single-dataset shape:\n `yo_help("repo/convert-single-dataset")`.\n\n## Add a dataset\n\n1. Make the directory and its dashboards folder:\n\n mkdir -p datasets/<name>/dashboards\n\n2. Write the model. Build it the way you build any model \u2014 compile a probe to\n read a schema, then write sources into files. `yo_help("develop/getting-started")`\n is the full procedure. Compile with paths relative to the REPO root:\n\n compile_file("datasets/<name>/orders.malloy")\n\n3. Write `datasets/<name>/index.malloy`, exporting what the dataset publishes:\n\n import { orders, PERIOD } from \'orders.malloy\'\n export { orders, PERIOD }\n\n Imports inside a dataset are relative to that dataset\'s directory, so\n `\'orders.malloy\'` is its sibling. Shared code one level up is\n `\'../../lib/orders.malloy\'`.\n\n4. Check the repo still publishes cleanly:\n\n malloyyo lint\n\n5. Publish:\n\n malloyyo publish --repo <owner/name> --create-datasets\n\n `--create-datasets` is only needed the first time a dataset appears.\n\n## Splitting one dataset into several\n\nOnly worth doing when the datasets really are separate \u2014 different people should\nsee different things, or one of them is scoped by who is asking and the others\nare not. A dataset is the unit of access: everything it publishes, a holder of\nits role gets.\n\nMove the sources for each into its own `datasets/<name>/`, put anything both need\nin `lib/`, and import it with `\'../../lib/<file>.malloy\'`.\n\n**Watch the givens.** A plain `import "../../lib/x.malloy"` brings that file\'s\n`given:` declarations with it, so a shared library that declares `MALLOYYO_EMAIL`\nmakes every dataset importing it scoped by the asker. Import selectively \u2014\n`import { orders } from \'../../lib/x.malloy\'` \u2014 when that is not what you want.\n',
|
|
219
216
|
"site/auto-update.md": '# Auto-update a data site weekly\n\nAdd a scheduled GitHub Actions job that rebuilds the data and commits it, so the\npublished Pages site stays current on its own. Use this **after** the site works\n(built with `yo_help("site/data-site")`). If the data never changes, skip this.\n\nThe worked example is `malloydata/malloyyo-imdb` \u2014 fetch and read its\n`scripts/build_data.sh` and `.github/workflows/refresh-data.yml`; adapt, don\'t\ncopy blindly.\n\n## Precondition: one command that rebuilds the data\n\nThe whole thing rests on a single script that turns the source URL(s) into the\ncommitted parquet \u2014 the same script you\'d run by hand. Keep it linear so it\nreads as a recipe:\n\n```bash\n# scripts/build_data.sh\nset -euo pipefail\ncd "$(dirname "$0")/.."\nbash data/get.sh # 1. download source URLs -> data/\nrm -f data/build.duckdb\nmalloy-cli -c malloy-build.json build transform.malloy # 2. transform\necho "ATTACH \'data/build.duckdb\' AS b (READ_ONLY); -- 3. export -> docs/*.parquet\n COPY b.thing TO \'docs/thing.parquet\' (FORMAT parquet)" | malloyyo sql\n```\n\n(If you used Path A \u2014 direct \u2014 the whole script is just the one\n`echo "COPY (FROM read_..(\'https://\u2026\')) TO \'docs/\u2026\'" | malloyyo sql` command.\nNo `data/get.sh`, `malloy-build.json`, `malloy-cli`, or transform needed \u2014 and\nno standalone `duckdb` either, since `malloyyo sql` uses the embedded one.)\n\nGitignore the working files: `data/*.gz`, `data/*.duckdb`, `MANIFESTS/`.\n\n## The workflow\n\n`.github/workflows/refresh-data.yml`:\n\n```yaml\nname: refresh-data\non:\n schedule:\n - cron: "0 6 * * 0" # weekly, Sunday 06:00 UTC\n workflow_dispatch: # ...and on demand from the Actions tab\njobs:\n refresh:\n runs-on: ubuntu-latest\n permissions:\n contents: write # so the job can commit + push\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-node@v4\n with: { node-version: "20" }\n # malloyyo carries its own DuckDB (`malloyyo sql`), so no duckdb CLI to install.\n # Add `@malloydata/cli` only for a `#@ persist` transform (malloy-cli build).\n - run: npm install -g @malloydata/malloyyo @malloydata/cli\n - uses: actions/setup-python@v5 # only if an enrichment step needs it\n with: { python-version: "3.12" }\n - name: Build data\n run: bash scripts/build_data.sh\n - name: Commit refreshed data\n run: |\n git config user.name github-actions\n git config user.email github-actions@github.com\n git add docs/*.parquet\n git commit -m "refresh data $(date -u +%F)" || exit 0 # unchanged = clean no-op\n git push\n```\n\n## Hard-won details (these came from a real first run)\n\n- **`git commit \u2026 || exit 0`** \u2014 an unchanged week means nothing new upstream.\n That is success, not a failed job.\n- **Secondary/enrichment steps must be best-effort.** If the build has a step\n that can fail independently (an external API lookup, an optional metadata\n enrichment) and it isn\'t essential to the data, mark it\n `continue-on-error: true`. Otherwise one flaky API call throws away an\n otherwise-good data refresh \u2014 the commit step never runs and the fresh parquet\n is lost. (In `malloyyo-imdb` this bit us: a poster-image lookup with a missing\n secret failed the whole job until it was made non-blocking.)\n- **Secrets** go in repo settings (Settings \u2192 Secrets and variables \u2192 Actions,\n or `gh secret set NAME`), referenced as `${{ secrets.NAME }}`. A step reading\n a secret that isn\'t set gets an empty string \u2014 pair that with best-effort.\n- **No re-bundle needed for data changes.** The dashboards fetch parquet at\n runtime, so committing fresh parquet updates the live site. Only install\n `@malloydata/malloyyo` and run `malloyyo dashboard bundle` in CI if the job\n also needs to regenerate the HTML (i.e. dashboard code changed \u2014 rare for a\n data refresh).\n- **Test before trusting the schedule.** Push, then Actions tab \u2192 the workflow \u2192\n **Run workflow** (that\'s `workflow_dispatch`). Or `gh workflow run\n refresh-data.yml`, then `gh run watch <id> --exit-status`.\n\n## Git growth, and the flatten\n\nEach committed refresh adds the full parquet to history (~its file size per\nrun). That is intentional simplicity, not a leak. When history gets too big,\nflatten it to a single commit by hand \u2014 **not** as part of the weekly job:\n\n```bash\ngit checkout --orphan flat && git add -A && git commit -m "flatten history"\ngit branch -M flat main && git push -f origin main\n```\n\nOnly do this on a repo you solely own (it force-pushes; other clones must\n`git reset --hard origin/main`). Run it as often or as rarely as you like.\nDocument this in the repo\'s README so future-you remembers it\'s available.\n\n## Done when\n\n- The workflow file is on the default branch (so `workflow_dispatch` shows up)\n- A manual run goes green end-to-end and commits fresh `docs/*.parquet`\n- The Pages site shows the new data\n',
|
|
220
217
|
"site/data-site.md": '# Build a public data site from a web data pointer\n\nGoal: given one or more **public data URLs**, produce a **GitHub Pages site** of\ninteractive Malloy dashboards that anyone can open in a browser. The data ends\nup as parquet in `docs/`; the dashboards are static HTML that query it\nclient-side with DuckDB-WASM \u2014 so there is no server, no database to run.\n\nTwo reference repos \u2014 read them, they are the ground truth for structure:\n\n- **`malloydata/malloyyo-babynames`** \u2014 the clean base pattern (one model file,\n a few dashboards, `docs/`). Data is static.\n- **`malloydata/malloyyo-imdb`** \u2014 the same, plus a `transform.malloy` that\n cleans raw source files into parquet, plus a **weekly auto-refresh** (that\n part is `yo_help("site/auto-update")`).\n\nFetch either with `gh api repos/<repo>/git/trees/HEAD?recursive=1` and read the\nfiles you need. Match their layout rather than inventing your own.\n\n## The shape of a finished repo\n\n```\nmalloy-config.json connection(s): duckdb (local), optionally md/gs mirrors\nindex.malloy the EXPORT SURFACE \u2014 only what this file exports is live\n<model>.malloy sources, measures, joins, givens (parameters)\nstorage.malloy sources point at the parquet \u2014 docs-local OR an https URL\n (or gs.malloy / md.malloy) same source names, swappable hosting (step 3)\ndashboards/*.malloy the query behind each dashboard\ndashboards/*.jsx the dashboard layout (grid of charts/tables)\ndocs/ PUBLISHED SITE \u2014 bundled HTML (+ the *.parquet if docs-local)\n *.parquet the data, when hosted docs-local (committed; served by Pages)\n *.html .nojekyll written by `malloyyo dashboard bundle`\n.mcp.json written by `malloyyo init` (author-mode Claude)\n```\n\n## Prerequisites (install once)\n\n```bash\nnpm install -g @malloydata/malloyyo # the `malloyyo` command\n```\n\nThat\'s the whole toolchain for the common case. `malloyyo` has **DuckDB built\nin** \u2014 `malloyyo sql` runs SQL (read a URL, `COPY` to parquet) through it, so you\ndo **not** need a standalone `duckdb` CLI.\n\nAdd `@malloydata/cli` (`malloy-cli`) **only** if you do a heavier transform with\nMalloy `#@ persist` (see Path B in `yo_help("site/data-to-parquet")`):\n\n```bash\nnpm install -g @malloydata/cli # only for #@ persist transforms\n```\n\n## Recipe\n\nWork top to bottom. After each step, prove it before moving on.\n\n### 1. Scaffold\n\n```bash\nmkdir my-site && cd my-site && git init\nmalloyyo init # writes .mcp.json (author-mode Claude) + index.malloy stub\n```\n\nCreate `malloy-config.json` with a local DuckDB connection (this is what reads\nthe parquet):\n\n```json\n{ "connections": { "duckdb": { "is": "duckdb" } } }\n```\n\n### 2. Get the web data into `docs/*.parquet`\n\nThis is the one dataset-specific step. Two paths \u2014 pick the simpler one that\nworks. Full detail and copy-paste commands: **`yo_help("site/data-to-parquet")`**.\n\n- **Direct** (default): `malloyyo sql` reads the URL and writes parquet in one\n shot, through the embedded DuckDB:\n ```bash\n echo "COPY (FROM read_csv_auto(\'https://\u2026/thing.csv\')) TO \'docs/thing.parquet\'" | malloyyo sql\n ```\n Use when the web data is already close to what you want to show.\n- **Transform** (when you need to clean / join / rank / reshape): write a\n `transform.malloy` with `#@ persist` sources and build it with `malloy-cli`,\n then export the tables to `docs/*.parquet`. This is the `malloyyo-imdb`\n pattern.\n\nVerify: `malloyyo sql -e "DESCRIBE SELECT * FROM \'docs/thing.parquet\'"` and a\n`SELECT count(*)`.\n\n### 3. Write the model\n\nA storage file \u2014 one source per parquet file \u2014 that the rest of the model builds\non. **Where the parquet lives is a choice** (the two examples differ here):\n\n- **docs-local** (the `malloyyo-imdb` way) \u2014 parquet committed in `docs/`,\n served same-origin by Pages. Self-contained; git carries the data. Address it\n by project-relative path so the same spelling works locally and published:\n ```malloy\n // storage.malloy\n source: thing_table is duckdb.table(\'docs/thing.parquet\') extend {}\n ```\n- **External URL** (the `malloyyo-babynames` way \u2014 it uses `import "gs.malloy"`)\n \u2014 parquet hosted on GCS / a CDN / any https URL; `docs/` holds only the HTML,\n so git stays small. DuckDB-WASM fetches the URL at runtime:\n ```malloy\n // gs.malloy\n source: thing_table is duckdb.table(\'https://storage.googleapis.com/\u2026/thing.parquet\') extend {}\n ```\n\nKeep the **source names identical** across storage files (as babynames does with\n`gs.malloy` / `md.malloy`) so switching hosting is a one-line import change in\nthe model. Start docs-local (simplest); move to a URL if git growth bites.\n\n`<model>.malloy` \u2014 build real sources on top: primary keys, measures, joins,\nand **givens** (the parameters that become the dashboard\'s filter controls).\n`index.malloy` \u2014 re-export exactly the sources/queries/givens the dashboards\nuse. **Only what `index.malloy` exports is visible** to dashboards, `dashboard\ndev`, and the hosted app.\n\nGive each exported source a `#"` doc string. It\'s what `list_sources` shows the\nhosted app and any MCP client, and it\'s the only place to record the things a\nconsumer can\'t infer \u2014 which source answers which question, and any measure\nwhose meaning is subtle.\n\nFor Malloy modeling and givens specifics, lean on the author MCP rather than\nguessing: the repo\'s `.mcp.json` wires `mcp__malloyyo_author__*`. Call\n`mcp__malloyyo_author__compile` to check files and\n`mcp__malloyyo_author__yo_help` for topics (`develop/working-with-models`).\n\nThree shapes that compile, pass `lint`, and still produce a dashboard that looks\nright and isn\'t:\n\n- **A stage-level `where:` zeroes measures that carry their own filters.**\n `count() { where: status = \'Sold\' }` needs the sold rows to reach it; a\n `where: in_cellar` on the stage removes them first and the measure reads 0\n everywhere. Constrain the output with `having:` instead.\n- **Grouping finer than the fact scatters the counts.** Group by an attribute\n that only applies to some rows and the measures over the others split across\n groups and read near-zero. Pick the grain the question is asked at; if you\n need both, group at the coarser one and `nest:` the finer.\n- **Detail tables default to raw field names and bare numbers.** `exit_date` and\n `829.57`, and a null renders as `\u2205`. Tag them \u2014 `# label="Opened"`,\n `# label="Paid" currency` \u2014 and `coalesce(field, \'\')` where empty is\n meaningful rather than missing.\n\nAggregate locality is NOT one of these: Malloy\'s symmetric aggregates keep\nmeasures honest through a `join_many` fan-out, and it warns (or errors) rather\nthan silently answering when an `avg`/`sum` crosses a join without explicit\nlocality. `notes.rating.avg()` already averages at the notes grain; ask for\n`source.avg(notes.rating)` only when you want the fan-out-weighted number. See\n`yo_help language/aggregate-locality-symmetric-aggregates`.\n\n### 4. Author dashboards and preview live\n\nEach dashboard is **one self-contained `dashboards/<name>.malloy`** \u2014 the query,\nits filtering, and the tags that lay it out. The filename is the dashboard\'s\nname and URL slug.\n\n**A `.jsx` is OPTIONAL and most dashboards don\'t need one.** The default form \u2014\nan inline query tagged `# artifact` + `# dashboard { columns=6 }` \u2014 renders KPI\ntiles and a card grid with no React at all. Reach for a sibling\n`dashboards/<name>.jsx` only when you want bespoke layout or presentation the\ntags can\'t express (a hand-built card, a picker, a Vega chart).\n\nAuthor with the `malloyyo_author` MCP and its `yo_help` topics \u2014 **read these,\ndon\'t guess the API**: `dashboards/authoring` (the dashboard file),\n`dashboards/grid-layout` (colspans), `dashboards/charts` (chart tags and their\nchannel rules), `dashboards/givens-and-controls` (filters),\n`dashboards/custom-components` and `dashboards/vega-charts` (only if you add a\n`.jsx`).\n\nPreview in a browser with live reload:\n\n```bash\nmalloyyo dashboard dev # serves at http://localhost:4173\n```\n\nTwo things about `dashboard dev` worth knowing:\n\n- It runs the queries **server-side**, through the same local DuckDB the CLI\n uses \u2014 the browser only renders results. So it works without DuckDB-WASM or\n its CDN extensions, and it is the only way to see a custom component render\n against real data short of publishing.\n- It holds **one long-lived connection set with a warm schema cache** (a\n deliberate tradeoff \u2014 refetching schemas per compile reads like a hang on a\n warehouse). The file watcher hot-reloads your `.malloy` edits and re-discovers\n `# artifact` tags, but nothing invalidates that cache, so a change to the\n DATA\'s shape \u2014 a new column in a parquet \u2014 needs a restart. The tell is\n `\'<column>\' is not defined` in the browser while the CLI and `lint` are green.\n\nIterate here until the dashboards look right. **Look at every dashboard before\nyou call it done**: `malloyyo lint` compiles the dashboards against the model but\ndoes not render them, so it cannot see a chart that throws at render time, a\ncomponent whose CSS resolved to nothing, or a column of raw field names.\n`malloyyo test` previews what the hosted claude.ai app would see.\n\n### 5. Build the static site\n\n```bash\nmalloyyo dashboard bundle --out docs\n# optional: add analytics + a title\nmalloyyo dashboard bundle --out docs --title "My Site" --analytics G-XXXXXXXXXX\n```\n\n`bundle` writes the HTML, `.nojekyll`, and assets into `docs/`, next to the\nparquet from step 2. The pages fetch `./*.parquet` relative to the site root, so\nthe data must already be in `docs/` \u2014 always run the data step before bundling.\n\n### 6. Publish on GitHub Pages\n\nCommit `docs/` and turn on Pages (Settings \u2192 Pages \u2192 Deploy from a branch \u2192\n`main` / `docs`). Full steps + gotchas: **`yo_help("site/github-pages")`**.\n\n### 7. (Optional) Keep it fresh automatically\n\nIf the data comes from a URL that updates over time and you want the site to\ntrack it, add a weekly GitHub Actions refresh \u2014 see\n**`yo_help("site/auto-update")`** (worked example: `malloydata/malloyyo-imdb`).\n\n## Done when\n\n- `malloyyo sql -e "SELECT count(*) FROM \'docs/<file>.parquet\'"` returns real rows\n- `malloyyo lint` passes\n- **you have LOOKED at every dashboard** in `malloyyo dashboard dev` \u2014 each one\n renders with no console errors, every tile has data, and the labels read like\n English rather than column names. `lint` passing is not this; a dashboard can\n compile and still throw at render time.\n- `docs/` has the bundled `*.html` + `.nojekyll` alongside the parquet\n- the Pages URL loads and the dashboards populate (data fetch succeeds)\n',
|
|
221
218
|
"site/data-to-parquet.md": '# Getting web data into `docs/*.parquet`\n\nThe site queries parquet files (in `docs/`, or at an https URL \u2014 see the model\nstep). This is how you turn a public data URL into that parquet. Pick the\nsimplest path that produces clean data.\n\n`malloyyo sql` runs SQL through malloyyo\'s **embedded DuckDB** \u2014 no standalone\n`duckdb` binary. It takes SQL from `-e`, `-f <file>`, or stdin, and runs against\na connection from `malloy-config.json` (default `duckdb`).\n\n## Path A \u2014 Direct (default)\n\nOne command reads the URL and writes parquet. Best when the web data is already\nclose to what you want to display.\n\n```bash\n# CSV / TSV\necho "COPY (SELECT * FROM read_csv_auto(\'https://example.com/data.csv\'))\n TO \'docs/data.parquet\' (FORMAT parquet)" | malloyyo sql\n\n# TSV, gzipped, tab-delimited, header row (IMDb-style)\necho "COPY (SELECT * FROM read_csv_auto(\'https://example.com/data.tsv.gz\',\n delim=\'\\t\', header=true, all_varchar=true))\n TO \'docs/data.parquet\' (FORMAT parquet)" | malloyyo sql\n\n# JSON / NDJSON\necho "COPY (SELECT * FROM read_json_auto(\'https://example.com/data.json\'))\n TO \'docs/data.parquet\' (FORMAT parquet)" | malloyyo sql\n\n# Already parquet on the web \u2014 just fetch it\ncurl -fsSL -o docs/data.parquet \'https://example.com/data.parquet\'\n```\n\nFor anything longer than a line or two, put the SQL in a file and run it with\n`-f`, so it\'s re-runnable and diffable:\n\n```bash\nmalloyyo sql -f scripts/build.sql # a file of ;-separated statements\n```\n\nNotes:\n- DuckDB autoloads `httpfs` for `https://` reads.\n- Keep only the columns/rows the dashboards need \u2014 `SELECT` the columns and add a\n `WHERE` to drop noise. Smaller parquet = faster page loads.\n- Cast types here if the source is all-strings: `col::INT`, `col::DOUBLE`, etc.\n\n## Path B \u2014 Transform with Malloy (when you need to reshape)\n\nUse when the data needs cleaning, joining across files, ranking, or nesting \u2014\nthe `malloydata/malloyyo-imdb` case. You write the transform once in Malloy;\n`malloy-cli build` materializes it; you export the tables to parquet. This path\nalso needs `@malloydata/cli` (`npm install -g @malloydata/cli`).\n\n**1. A build connection** \u2014 `malloy-build.json` (a DuckDB db just for building):\n\n```json\n{ "connections": { "build": { "is": "duckdb", "databasePath": "data/build.duckdb" } } }\n```\n\n**2. `transform.malloy`** \u2014 read the raw source(s) through the `build`\nconnection and mark each output source with `#@ persist name="\u2026"`:\n\n```malloy\n##! experimental.persistence experimental.virtual_source\n\nsource: raw is build.sql("""\n SELECT * FROM read_csv_auto(\'data/raw.csv.gz\', delim=\'\\t\', all_varchar=true, header=true)\n""")\n\n#@ persist name="thing"\nsource: thing_base is raw -> {\n where: some_count::number > 100\n select: id, name, value is value::number\n calculate: rank is rank() { order_by: value::number desc }\n}\n```\n\nDownload the raw files first (a `data/get.sh` that `wget`s the URLs into `data/`,\ngitignored). Gitignore `data/`, `data/*.duckdb`, and `MANIFESTS/`.\n\n**3. Build, then export the persisted tables to `docs/`** \u2014 the export still\ngoes through `malloyyo sql` (no standalone duckdb needed):\n\n```bash\nrm -f data/build.duckdb\nmalloy-cli -c malloy-build.json build transform.malloy # -> tables in build.duckdb\necho "ATTACH \'data/build.duckdb\' AS b (READ_ONLY);\n COPY b.thing TO \'docs/thing.parquet\' (FORMAT parquet)" | malloyyo sql\n```\n\nThe persist `name=` is the table name inside `build.duckdb`; the `COPY` names the\nserved file (they can differ \u2014 e.g. `thing` \u2192 `docs/mysite_thing.parquet`).\n\nWrap steps 1\u20133 in a single `scripts/build_data.sh` so it is one command, by hand\nand in CI. That\'s exactly what `yo_help("site/auto-update")` automates weekly.\n\n## Either way, verify\n\n```bash\nmalloyyo sql -e "DESCRIBE SELECT * FROM \'docs/thing.parquet\'"\nmalloyyo sql -e "SELECT count(*) FROM \'docs/thing.parquet\'"\n```\n\nThen point a storage source at it (docs-local shown; an https URL works too \u2014\nsee the model step in SKILL.md):\n\n```malloy\nsource: thing_table is duckdb.table(\'docs/thing.parquet\') extend {}\n```\n',
|
|
@@ -1051,13 +1048,24 @@ function modelCatalogEntry(model_ref, model) {
|
|
|
1051
1048
|
if (sources.length) entry.sources = sources;
|
|
1052
1049
|
return entry;
|
|
1053
1050
|
}
|
|
1051
|
+
var RESERVED_GIVEN_PREFIX = "MALLOYYO_";
|
|
1052
|
+
function withoutReservedGivens(givens) {
|
|
1053
|
+
if (!givens) return void 0;
|
|
1054
|
+
const kept = {};
|
|
1055
|
+
for (const [name, value] of Object.entries(givens)) {
|
|
1056
|
+
if (!name.startsWith(RESERVED_GIVEN_PREFIX)) kept[name] = value;
|
|
1057
|
+
}
|
|
1058
|
+
return kept;
|
|
1059
|
+
}
|
|
1054
1060
|
function unreferencedGivens(required, referenced) {
|
|
1055
1061
|
if (required.length === 0) return [];
|
|
1056
1062
|
const seen = new Set(referenced);
|
|
1057
|
-
|
|
1063
|
+
if (required.some((name) => seen.has(name))) return [];
|
|
1064
|
+
return [...required].sort();
|
|
1058
1065
|
}
|
|
1059
1066
|
function unreferencedGivensMessage(missing) {
|
|
1060
|
-
|
|
1067
|
+
const many = missing.length > 1;
|
|
1068
|
+
return `${missing.join(", ")}: this data is scoped by ${many ? "these givens" : "this given"}, and the query references ${many ? "none of them" : "it nowhere"}. A query here must run against a source that filters on ${many ? "one of them" : "it"} \u2014 put the filter on the source (\`source: x is \u2026 extend { where: owner = $${missing[0]} }\`) so every query over it carries the scope.`;
|
|
1061
1069
|
}
|
|
1062
1070
|
function declaredGivenNames(model) {
|
|
1063
1071
|
return new Set(model.givens?.keys() ?? []);
|
|
@@ -1108,7 +1116,8 @@ async function executeMaterialized(query, opts, loadProblems, decorate = (p) =>
|
|
|
1108
1116
|
};
|
|
1109
1117
|
}
|
|
1110
1118
|
}
|
|
1111
|
-
const
|
|
1119
|
+
const caller = withoutReservedGivens(opts.givens);
|
|
1120
|
+
const compileOpts = caller ? { givens: caller } : void 0;
|
|
1112
1121
|
try {
|
|
1113
1122
|
const t0 = Date.now();
|
|
1114
1123
|
const sql = (await query.getSQL(compileOpts)).trim();
|
|
@@ -1273,6 +1282,93 @@ async function runRestricted(runtime, entry, query, opts = {}) {
|
|
|
1273
1282
|
return { ok: false, problems: [...loadProblems, errorProblem(e, entry.href)] };
|
|
1274
1283
|
}
|
|
1275
1284
|
}
|
|
1285
|
+
function titleFromName(name) {
|
|
1286
|
+
return name.split(/[_-]+/).filter(Boolean).map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join(" ");
|
|
1287
|
+
}
|
|
1288
|
+
function datasetTitle(name, declared) {
|
|
1289
|
+
const t = declared?.trim();
|
|
1290
|
+
return t || titleFromName(name);
|
|
1291
|
+
}
|
|
1292
|
+
function readDatasetMeta(model) {
|
|
1293
|
+
const m = model;
|
|
1294
|
+
let title;
|
|
1295
|
+
try {
|
|
1296
|
+
const tag = m.annotations.parseAsTag().tag;
|
|
1297
|
+
if (tag.has("dataset")) {
|
|
1298
|
+
title = tag.tag("dataset")?.text("title") ?? tag.text("title");
|
|
1299
|
+
}
|
|
1300
|
+
} catch {
|
|
1301
|
+
}
|
|
1302
|
+
let description;
|
|
1303
|
+
try {
|
|
1304
|
+
const docs = m.annotations.forRoute('"').map((n) => n.content.trim()).filter(Boolean);
|
|
1305
|
+
if (docs.length) description = docs.join("\n");
|
|
1306
|
+
} catch {
|
|
1307
|
+
}
|
|
1308
|
+
return { ...title ? { title } : {}, ...description ? { description } : {} };
|
|
1309
|
+
}
|
|
1310
|
+
function nameToSlug(s) {
|
|
1311
|
+
return s.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "").slice(0, 48) || "dataset";
|
|
1312
|
+
}
|
|
1313
|
+
var DATASETS_DIR = "datasets";
|
|
1314
|
+
var ENTRY_FILE = "index.malloy";
|
|
1315
|
+
async function layoutFromListing(list, label = "this repo") {
|
|
1316
|
+
const root = await list("");
|
|
1317
|
+
if (root.length === 0) {
|
|
1318
|
+
return { ok: false, error: `${label} is empty or could not be read.` };
|
|
1319
|
+
}
|
|
1320
|
+
const hasRootEntry = root.some((e) => e.type === "file" && e.name === ENTRY_FILE);
|
|
1321
|
+
const hasDatasetsDir = root.some((e) => e.type === "dir" && e.name === DATASETS_DIR);
|
|
1322
|
+
if (hasRootEntry && hasDatasetsDir) {
|
|
1323
|
+
return {
|
|
1324
|
+
ok: false,
|
|
1325
|
+
error: `This repo has both a top-level ${ENTRY_FILE} and a ${DATASETS_DIR}/ directory, and those are two different layouts. Use one: keep ${ENTRY_FILE} at the root to publish a single dataset, or move it under ${DATASETS_DIR}/<name>/ and delete the root one to publish several.`
|
|
1326
|
+
};
|
|
1327
|
+
}
|
|
1328
|
+
if (hasRootEntry) return { ok: true, kind: "single" };
|
|
1329
|
+
if (!hasDatasetsDir) {
|
|
1330
|
+
return {
|
|
1331
|
+
ok: false,
|
|
1332
|
+
error: `No ${ENTRY_FILE} at the root of ${label}, and no ${DATASETS_DIR}/ directory either. A model repo needs one or the other.`
|
|
1333
|
+
};
|
|
1334
|
+
}
|
|
1335
|
+
const subdirs = (await list(DATASETS_DIR)).filter((e) => e.type === "dir");
|
|
1336
|
+
if (subdirs.length === 0) {
|
|
1337
|
+
return {
|
|
1338
|
+
ok: false,
|
|
1339
|
+
empty: true,
|
|
1340
|
+
error: `${DATASETS_DIR}/ has no subdirectories \u2014 each dataset is a directory under it.`
|
|
1341
|
+
};
|
|
1342
|
+
}
|
|
1343
|
+
const found = [];
|
|
1344
|
+
const missing = [];
|
|
1345
|
+
for (const d of subdirs) {
|
|
1346
|
+
const inside = await list(`${DATASETS_DIR}/${d.name}`);
|
|
1347
|
+
if (inside.some((e) => e.type === "file" && e.name === ENTRY_FILE)) {
|
|
1348
|
+
found.push({ name: nameToSlug(d.name), dir: `${DATASETS_DIR}/${d.name}` });
|
|
1349
|
+
} else {
|
|
1350
|
+
missing.push(`${DATASETS_DIR}/${d.name}`);
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
if (missing.length > 0) {
|
|
1354
|
+
return {
|
|
1355
|
+
ok: false,
|
|
1356
|
+
error: `${missing.join(", ")}: every directory under ${DATASETS_DIR}/ is a dataset and needs its own ${ENTRY_FILE}. Add one, or move the directory out of ${DATASETS_DIR}/ if it is shared code (a sibling \`lib/\` is imported by relative path).`
|
|
1357
|
+
};
|
|
1358
|
+
}
|
|
1359
|
+
const byName2 = /* @__PURE__ */ new Map();
|
|
1360
|
+
for (const d of found) byName2.set(d.name, [...byName2.get(d.name) ?? [], d.dir]);
|
|
1361
|
+
const collided = [...byName2.entries()].filter(([, dirs]) => dirs.length > 1);
|
|
1362
|
+
if (collided.length > 0) {
|
|
1363
|
+
return {
|
|
1364
|
+
ok: false,
|
|
1365
|
+
error: collided.map(([name, dirs]) => `${dirs.join(" and ")} both publish a dataset named "${name}"`).join("; ")
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1368
|
+
return { ok: true, kind: "multi", datasets: found.sort((a, b) => a.name.localeCompare(b.name)) };
|
|
1369
|
+
}
|
|
1370
|
+
var MAX_FILE = 4 * 1024 * 1024;
|
|
1371
|
+
var MAX_TOTAL = 64 * 1024 * 1024;
|
|
1276
1372
|
function renderGivenType2(t) {
|
|
1277
1373
|
if (t.type === "filter expression") {
|
|
1278
1374
|
return t.filterType ? `filter<${t.filterType}>` : "filter";
|
|
@@ -1539,9 +1635,9 @@ function prepareSource(base, input) {
|
|
|
1539
1635
|
const cached = cache.get(u.href);
|
|
1540
1636
|
if (cached !== void 0) return cached;
|
|
1541
1637
|
const text = await base.readURL(u);
|
|
1542
|
-
const
|
|
1543
|
-
cache.set(u.href,
|
|
1544
|
-
return
|
|
1638
|
+
const str2 = typeof text === "string" ? text : String(text);
|
|
1639
|
+
cache.set(u.href, str2);
|
|
1640
|
+
return str2;
|
|
1545
1641
|
}
|
|
1546
1642
|
};
|
|
1547
1643
|
return { reader, entry, readSource: (href) => cache.get(href) };
|
|
@@ -2127,11 +2223,112 @@ function developSurface(host, opts = {}) {
|
|
|
2127
2223
|
};
|
|
2128
2224
|
}
|
|
2129
2225
|
|
|
2130
|
-
// src/
|
|
2226
|
+
// src/discover.ts
|
|
2131
2227
|
import fs from "node:fs";
|
|
2132
2228
|
import path2 from "node:path";
|
|
2133
|
-
import
|
|
2229
|
+
import { fileURLToPath } from "node:url";
|
|
2134
2230
|
import { createRequire } from "node:module";
|
|
2231
|
+
var require2 = createRequire(import.meta.url);
|
|
2232
|
+
var HOST_LIBS = [
|
|
2233
|
+
"react",
|
|
2234
|
+
"react-dom",
|
|
2235
|
+
"react-dom/client",
|
|
2236
|
+
"react/jsx-runtime",
|
|
2237
|
+
"react/jsx-dev-runtime",
|
|
2238
|
+
"@malloydata/render",
|
|
2239
|
+
"@malloydata/malloy-filter"
|
|
2240
|
+
];
|
|
2241
|
+
var HOST_ALIAS = {};
|
|
2242
|
+
for (const spec of HOST_LIBS) {
|
|
2243
|
+
try {
|
|
2244
|
+
HOST_ALIAS[spec] = require2.resolve(spec);
|
|
2245
|
+
} catch {
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
function resolveRuntimeDir() {
|
|
2249
|
+
const candidates = [
|
|
2250
|
+
new URL("./frame-runtime/", import.meta.url),
|
|
2251
|
+
// src/dashboard.ts (tsx) OR dist/index.js (published)
|
|
2252
|
+
new URL("../src/frame-runtime/", import.meta.url)
|
|
2253
|
+
// built dist/ next to sibling src/ (checkout)
|
|
2254
|
+
].map((u) => fileURLToPath(u));
|
|
2255
|
+
const found = candidates.find((c) => fs.existsSync(c));
|
|
2256
|
+
if (!found) {
|
|
2257
|
+
throw new Error(
|
|
2258
|
+
"frame-runtime/ not found next to the CLI (looked in ./frame-runtime and ../src/frame-runtime). A published install should ship it in dist/; reinstall the CLI, or rebuild with `npm run build`."
|
|
2259
|
+
);
|
|
2260
|
+
}
|
|
2261
|
+
return found;
|
|
2262
|
+
}
|
|
2263
|
+
var hostAliasPlugin = {
|
|
2264
|
+
name: "host-alias",
|
|
2265
|
+
setup(b) {
|
|
2266
|
+
b.onResolve(
|
|
2267
|
+
{ filter: /^(react($|\/)|react-dom($|\/)|@malloydata\/(render|malloy-filter)$)/ },
|
|
2268
|
+
(args) => HOST_ALIAS[args.path] ? { path: HOST_ALIAS[args.path] } : void 0
|
|
2269
|
+
);
|
|
2270
|
+
}
|
|
2271
|
+
};
|
|
2272
|
+
async function discoverDashboards(root, runner) {
|
|
2273
|
+
const dir = path2.join(root, "dashboards");
|
|
2274
|
+
if (!fs.existsSync(dir)) return [];
|
|
2275
|
+
const files = fs.readdirSync(dir).filter((f) => f.endsWith(".malloy")).sort();
|
|
2276
|
+
const dashboards = [];
|
|
2277
|
+
for (const file of files) {
|
|
2278
|
+
const base = file.slice(0, -".malloy".length);
|
|
2279
|
+
const entryFile = path2.join("dashboards", file);
|
|
2280
|
+
const res = await runner.artifactForFile(entryFile, base);
|
|
2281
|
+
if (!res.ok) throw new Error(`dashboard ${file}: ${res.error}`);
|
|
2282
|
+
if (!res.artifact) continue;
|
|
2283
|
+
const component = ["jsx", "tsx"].map((ext) => path2.join(dir, `${base}.${ext}`)).find((p) => fs.existsSync(p));
|
|
2284
|
+
dashboards.push({ ...res.artifact, name: res.artifact.name || base, entryFile, tsxPath: component });
|
|
2285
|
+
}
|
|
2286
|
+
const about = aboutPage(root);
|
|
2287
|
+
if (about && !dashboards.some((d) => d.name === about.name)) dashboards.unshift(about);
|
|
2288
|
+
return dashboards;
|
|
2289
|
+
}
|
|
2290
|
+
function browserBuildBase() {
|
|
2291
|
+
const shims = path2.join(resolveRuntimeDir(), "..", "shims");
|
|
2292
|
+
return {
|
|
2293
|
+
platform: "browser",
|
|
2294
|
+
jsx: "automatic",
|
|
2295
|
+
loader: { ".css": "empty" },
|
|
2296
|
+
define: { "process.env.NODE_ENV": '"production"' },
|
|
2297
|
+
alias: {
|
|
2298
|
+
assert: path2.join(shims, "assert.cjs"),
|
|
2299
|
+
util: path2.join(shims, "util.cjs")
|
|
2300
|
+
},
|
|
2301
|
+
banner: {
|
|
2302
|
+
js: "globalThis.process||={env:{},platform:'browser',versions:{},argv:[],cwd:()=>'/'};"
|
|
2303
|
+
}
|
|
2304
|
+
};
|
|
2305
|
+
}
|
|
2306
|
+
var ABOUT_NAME = "index";
|
|
2307
|
+
var ABOUT_TITLE = "About";
|
|
2308
|
+
function aboutPage(root) {
|
|
2309
|
+
const dir = path2.join(root, "dashboards");
|
|
2310
|
+
if (fs.existsSync(path2.join(dir, "index.malloy"))) return null;
|
|
2311
|
+
const component = ["jsx", "tsx"].map((ext) => path2.join(dir, `index.${ext}`)).find((p) => fs.existsSync(p));
|
|
2312
|
+
if (!component) return null;
|
|
2313
|
+
return { name: ABOUT_NAME, query: "", title: ABOUT_TITLE, tsxPath: component };
|
|
2314
|
+
}
|
|
2315
|
+
var rendersNoData = (d) => !d.query && (!d.tiles || d.tiles.length === 0);
|
|
2316
|
+
|
|
2317
|
+
// src/host.ts
|
|
2318
|
+
import fs3 from "node:fs";
|
|
2319
|
+
import path4 from "node:path";
|
|
2320
|
+
import url3 from "node:url";
|
|
2321
|
+
import {
|
|
2322
|
+
MalloyConfig,
|
|
2323
|
+
Runtime,
|
|
2324
|
+
discoverConfig
|
|
2325
|
+
} from "@malloydata/malloy";
|
|
2326
|
+
|
|
2327
|
+
// src/connections.ts
|
|
2328
|
+
import fs2 from "node:fs";
|
|
2329
|
+
import path3 from "node:path";
|
|
2330
|
+
import url2 from "node:url";
|
|
2331
|
+
import { createRequire as createRequire2 } from "node:module";
|
|
2135
2332
|
import { getRegisteredConnectionTypes } from "@malloydata/malloy";
|
|
2136
2333
|
var CONNECTIONS_PKG = "@malloydata/malloy-connections";
|
|
2137
2334
|
var CONNECTOR_IMPORTS = [
|
|
@@ -2163,13 +2360,13 @@ function indent(text, by = " ") {
|
|
|
2163
2360
|
return text.split("\n").map((l) => l ? by + l : l).join("\n");
|
|
2164
2361
|
}
|
|
2165
2362
|
function requireFrom(dir) {
|
|
2166
|
-
return
|
|
2363
|
+
return createRequire2(path3.join(dir, "__resolve__.cjs"));
|
|
2167
2364
|
}
|
|
2168
2365
|
function packageRootOf(entry) {
|
|
2169
|
-
let dir =
|
|
2366
|
+
let dir = path3.dirname(entry);
|
|
2170
2367
|
for (; ; ) {
|
|
2171
|
-
if (
|
|
2172
|
-
const up =
|
|
2368
|
+
if (fs2.existsSync(path3.join(dir, "package.json"))) return dir;
|
|
2369
|
+
const up = path3.dirname(dir);
|
|
2173
2370
|
if (up === dir) return null;
|
|
2174
2371
|
dir = up;
|
|
2175
2372
|
}
|
|
@@ -2177,26 +2374,26 @@ function packageRootOf(entry) {
|
|
|
2177
2374
|
function packageDir(spec, from) {
|
|
2178
2375
|
const req = requireFrom(from);
|
|
2179
2376
|
try {
|
|
2180
|
-
return
|
|
2377
|
+
return fs2.realpathSync(path3.dirname(req.resolve(`${spec}/package.json`)));
|
|
2181
2378
|
} catch {
|
|
2182
2379
|
}
|
|
2183
2380
|
try {
|
|
2184
2381
|
const root = packageRootOf(req.resolve(spec));
|
|
2185
|
-
return root ?
|
|
2382
|
+
return root ? fs2.realpathSync(root) : null;
|
|
2186
2383
|
} catch {
|
|
2187
2384
|
return null;
|
|
2188
2385
|
}
|
|
2189
2386
|
}
|
|
2190
2387
|
function readVersion(pkgDir) {
|
|
2191
2388
|
try {
|
|
2192
|
-
const raw =
|
|
2389
|
+
const raw = fs2.readFileSync(path3.join(pkgDir, "package.json"), "utf8");
|
|
2193
2390
|
return String(JSON.parse(raw).version ?? "?");
|
|
2194
2391
|
} catch {
|
|
2195
2392
|
return "?";
|
|
2196
2393
|
}
|
|
2197
2394
|
}
|
|
2198
2395
|
function selfDir() {
|
|
2199
|
-
return
|
|
2396
|
+
return path3.dirname(url2.fileURLToPath(import.meta.url));
|
|
2200
2397
|
}
|
|
2201
2398
|
function malloyCopies(from = selfDir()) {
|
|
2202
2399
|
const here = from;
|
|
@@ -2362,6 +2559,7 @@ async function withConnectionDiagnostics(fn) {
|
|
|
2362
2559
|
|
|
2363
2560
|
// src/server-givens.ts
|
|
2364
2561
|
var RESERVED_PREFIX = "MALLOYYO_";
|
|
2562
|
+
var LIST_GIVENS = /* @__PURE__ */ new Set([`${RESERVED_PREFIX}ROLES`]);
|
|
2365
2563
|
function serverFilledNames(declared) {
|
|
2366
2564
|
return [...declared].filter((n) => n.startsWith(RESERVED_PREFIX)).sort();
|
|
2367
2565
|
}
|
|
@@ -2369,13 +2567,18 @@ function readServerGivens(declared, env = process.env) {
|
|
|
2369
2567
|
const out = { givens: {}, missing: [] };
|
|
2370
2568
|
for (const name of serverFilledNames(declared)) {
|
|
2371
2569
|
const value = env[name]?.trim();
|
|
2372
|
-
if (value)
|
|
2373
|
-
|
|
2570
|
+
if (!value) {
|
|
2571
|
+
out.missing.push(name);
|
|
2572
|
+
continue;
|
|
2573
|
+
}
|
|
2574
|
+
out.givens[name] = LIST_GIVENS.has(name) ? value.split(",").map((v) => v.trim()).filter(Boolean) : value;
|
|
2374
2575
|
}
|
|
2375
2576
|
return out;
|
|
2376
2577
|
}
|
|
2377
2578
|
function missingServerGivensMessage(missing) {
|
|
2378
|
-
const example = missing.map(
|
|
2579
|
+
const example = missing.map(
|
|
2580
|
+
(n) => n.endsWith("_EMAIL") ? `${n}=you@example.com` : LIST_GIVENS.has(n) ? `${n}=finance,sales` : `${n}=\u2026`
|
|
2581
|
+
).join(" ");
|
|
2379
2582
|
return `${missing.join(", ")}: filled by a Malloyyo server from whoever is signed in, and unset here. This model is scoped by ${missing.length > 1 ? "them" : "it"}, so running without ${missing.length > 1 ? "them" : "it"} would show nothing at all. Set ${missing.length > 1 ? "them" : "it"} to whoever you want to look at the data as:
|
|
2380
2583
|
${example} malloyyo dashboard dev`;
|
|
2381
2584
|
}
|
|
@@ -2409,28 +2612,29 @@ function fsReader() {
|
|
|
2409
2612
|
if (u.protocol !== "file:") {
|
|
2410
2613
|
throw new Error(`unsupported URL scheme for import: ${u.href}`);
|
|
2411
2614
|
}
|
|
2412
|
-
return
|
|
2615
|
+
return fs3.promises.readFile(u, "utf8");
|
|
2413
2616
|
}
|
|
2414
2617
|
};
|
|
2415
2618
|
}
|
|
2416
|
-
async function loadConfig(rootUrl, reader) {
|
|
2417
|
-
const discovered = await discoverConfig(rootUrl,
|
|
2619
|
+
async function loadConfig(rootUrl, reader, ceilingUrl = rootUrl) {
|
|
2620
|
+
const discovered = await discoverConfig(rootUrl, ceilingUrl, reader).catch(() => null);
|
|
2418
2621
|
return discovered ?? new MalloyConfig({ includeDefaultConnections: true }, {
|
|
2419
2622
|
rootDirectory: rootUrl.toString()
|
|
2420
2623
|
});
|
|
2421
2624
|
}
|
|
2422
|
-
async function makeRunner(root) {
|
|
2625
|
+
async function makeRunner(root, opts = {}) {
|
|
2423
2626
|
await initConnections();
|
|
2424
|
-
const abs =
|
|
2425
|
-
const rootUrl = url3.pathToFileURL(abs +
|
|
2627
|
+
const abs = path4.resolve(root);
|
|
2628
|
+
const rootUrl = url3.pathToFileURL(abs + path4.sep);
|
|
2426
2629
|
const reader = fsReader();
|
|
2427
2630
|
let configPromise = null;
|
|
2428
|
-
const
|
|
2631
|
+
const ceilingUrl = opts.repoRoot ? url3.pathToFileURL(path4.resolve(opts.repoRoot) + path4.sep) : rootUrl;
|
|
2632
|
+
const getConfig = () => configPromise ??= loadConfig(rootUrl, reader, ceilingUrl);
|
|
2429
2633
|
let scopePromise = null;
|
|
2430
2634
|
const givenScope = () => scopePromise ??= (async () => {
|
|
2431
|
-
if (!
|
|
2635
|
+
if (!fs3.existsSync(path4.join(abs, ENTRY))) return { givens: {}, finalize: [], missing: [] };
|
|
2432
2636
|
const config = await getConfig();
|
|
2433
|
-
const { reader: prepared, entry } = prepareSource(reader, { url:
|
|
2637
|
+
const { reader: prepared, entry } = prepareSource(reader, { url: path4.join(abs, ENTRY) });
|
|
2434
2638
|
let model;
|
|
2435
2639
|
try {
|
|
2436
2640
|
model = await new Runtime({ config, urlReader: prepared }).loadModel(entry).getModel();
|
|
@@ -2461,7 +2665,7 @@ async function makeRunner(root) {
|
|
|
2461
2665
|
};
|
|
2462
2666
|
async function leaseIn(entryFile, fn) {
|
|
2463
2667
|
const config = await getConfig();
|
|
2464
|
-
const { reader: prepared, entry } = prepareSource(reader, { url:
|
|
2668
|
+
const { reader: prepared, entry } = prepareSource(reader, { url: path4.join(abs, entryFile) });
|
|
2465
2669
|
const { givens: scoped } = await givenScope();
|
|
2466
2670
|
const runtime = new Runtime({
|
|
2467
2671
|
config,
|
|
@@ -2485,7 +2689,18 @@ async function makeRunner(root) {
|
|
|
2485
2689
|
};
|
|
2486
2690
|
return {
|
|
2487
2691
|
root: abs,
|
|
2488
|
-
entryExists: () =>
|
|
2692
|
+
entryExists: () => fs3.existsSync(path4.join(abs, ENTRY)),
|
|
2693
|
+
datasetMeta: async () => {
|
|
2694
|
+
if (!fs3.existsSync(path4.join(abs, ENTRY))) return {};
|
|
2695
|
+
try {
|
|
2696
|
+
const config = await getConfig();
|
|
2697
|
+
const { reader: prepared, entry } = prepareSource(reader, { url: path4.join(abs, ENTRY) });
|
|
2698
|
+
const model = await new Runtime({ config, urlReader: prepared }).loadModel(entry).getModel();
|
|
2699
|
+
return readDatasetMeta(model);
|
|
2700
|
+
} catch {
|
|
2701
|
+
return {};
|
|
2702
|
+
}
|
|
2703
|
+
},
|
|
2489
2704
|
async dispose() {
|
|
2490
2705
|
clearIdleTimer();
|
|
2491
2706
|
if (!configPromise) return;
|
|
@@ -2579,99 +2794,99 @@ async function makeRunner(root) {
|
|
|
2579
2794
|
};
|
|
2580
2795
|
}
|
|
2581
2796
|
|
|
2582
|
-
// src/
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
"
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
"
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
}
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
throw new Error(
|
|
2614
|
-
"frame-runtime/ not found next to the CLI (looked in ./frame-runtime and ../src/frame-runtime). A published install should ship it in dist/; reinstall the CLI, or rebuild with `npm run build`."
|
|
2615
|
-
);
|
|
2616
|
-
}
|
|
2617
|
-
return found;
|
|
2797
|
+
// src/repo.ts
|
|
2798
|
+
var SKIP_DIRS = /* @__PURE__ */ new Set(["node_modules", ".git"]);
|
|
2799
|
+
var BUNDLE_MARKER = ["assets", "model-files.js"];
|
|
2800
|
+
function isBundleOutput(dir) {
|
|
2801
|
+
return fs4.existsSync(path5.join(dir, ...BUNDLE_MARKER));
|
|
2802
|
+
}
|
|
2803
|
+
function repoRootOf(modelRoot) {
|
|
2804
|
+
const abs = path5.resolve(modelRoot);
|
|
2805
|
+
if (fs4.existsSync(path5.join(abs, ".git"))) return abs;
|
|
2806
|
+
const parent = path5.dirname(abs);
|
|
2807
|
+
if (path5.basename(parent) !== DATASETS_DIR) return abs;
|
|
2808
|
+
const above = path5.dirname(parent);
|
|
2809
|
+
const isRepo = fs4.existsSync(path5.join(above, ".git")) || fs4.existsSync(path5.join(above, "malloy-config.json"));
|
|
2810
|
+
return isRepo ? above : abs;
|
|
2811
|
+
}
|
|
2812
|
+
function fsLister(root) {
|
|
2813
|
+
return async (p) => {
|
|
2814
|
+
const dir = p ? path5.join(root, p) : root;
|
|
2815
|
+
if (!fs4.existsSync(dir)) return [];
|
|
2816
|
+
let entries;
|
|
2817
|
+
try {
|
|
2818
|
+
entries = fs4.readdirSync(dir, { withFileTypes: true });
|
|
2819
|
+
} catch {
|
|
2820
|
+
return [];
|
|
2821
|
+
}
|
|
2822
|
+
return entries.filter((e) => e.isDirectory() || e.isFile()).map((e) => ({
|
|
2823
|
+
name: e.name,
|
|
2824
|
+
path: p ? `${p}/${e.name}` : e.name,
|
|
2825
|
+
type: e.isDirectory() ? "dir" : "file"
|
|
2826
|
+
}));
|
|
2827
|
+
};
|
|
2618
2828
|
}
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
}
|
|
2627
|
-
};
|
|
2628
|
-
async function discoverDashboards(root, runner) {
|
|
2629
|
-
const dir = path4.join(root, "dashboards");
|
|
2630
|
-
if (!fs3.existsSync(dir)) return [];
|
|
2631
|
-
const files = fs3.readdirSync(dir).filter((f) => f.endsWith(".malloy")).sort();
|
|
2829
|
+
async function discoverRepoDashboards(root) {
|
|
2830
|
+
const repoRoot = repoRootOf(root);
|
|
2831
|
+
const layout = await layoutFromListing(fsLister(root), root);
|
|
2832
|
+
if (!layout.ok) throw new Error(layout.error);
|
|
2833
|
+
const units = layout.kind === "single" ? [{ dataset: "", dir: "" }] : layout.datasets.map((d) => ({ dataset: d.name, dir: d.dir }));
|
|
2834
|
+
const multi = units.length > 1 || units.some((u) => u.dir !== "");
|
|
2835
|
+
const runners = [];
|
|
2632
2836
|
const dashboards = [];
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
const
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2837
|
+
const titleOf = /* @__PURE__ */ new Map();
|
|
2838
|
+
try {
|
|
2839
|
+
for (const u of units) {
|
|
2840
|
+
const modelRoot = u.dir ? path5.join(root, u.dir) : root;
|
|
2841
|
+
const runner = await makeRunner(modelRoot, { repoRoot });
|
|
2842
|
+
runners.push(runner);
|
|
2843
|
+
if (!runner.entryExists()) {
|
|
2844
|
+
throw new Error(
|
|
2845
|
+
multi ? `${u.dir}/index.malloy is missing \u2014 every directory under datasets/ is a dataset and needs one.` : `No index.malloy at ${root} \u2014 run this from a Malloy model repo, or ask claude to add a dataset.`
|
|
2846
|
+
);
|
|
2847
|
+
}
|
|
2848
|
+
if (u.dataset) titleOf.set(u.dataset, (await runner.datasetMeta()).title);
|
|
2849
|
+
for (const d of await discoverDashboards(modelRoot, runner)) {
|
|
2850
|
+
dashboards.push({
|
|
2851
|
+
...d,
|
|
2852
|
+
dataset: u.dataset,
|
|
2853
|
+
datasetLabel: u.dataset ? datasetTitle(u.dataset, titleOf.get(u.dataset)) : "",
|
|
2854
|
+
slug: multi ? `${u.dataset}/${d.name}` : d.name,
|
|
2855
|
+
runner,
|
|
2856
|
+
root: modelRoot,
|
|
2857
|
+
repoEntryFile: d.entryFile && u.dir ? `${u.dir}/${d.entryFile}` : d.entryFile
|
|
2858
|
+
});
|
|
2859
|
+
}
|
|
2860
|
+
}
|
|
2861
|
+
} catch (e) {
|
|
2862
|
+
await Promise.all(runners.map((r) => r.dispose().catch(() => {
|
|
2863
|
+
})));
|
|
2864
|
+
throw e;
|
|
2641
2865
|
}
|
|
2642
|
-
const about = aboutPage(root);
|
|
2643
|
-
if (about && !dashboards.some((d) => d.name === about.name)) dashboards.unshift(about);
|
|
2644
|
-
return dashboards;
|
|
2645
|
-
}
|
|
2646
|
-
function browserBuildBase() {
|
|
2647
|
-
const shims = path4.join(resolveRuntimeDir(), "..", "shims");
|
|
2648
2866
|
return {
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
assert: path4.join(shims, "assert.cjs"),
|
|
2655
|
-
util: path4.join(shims, "util.cjs")
|
|
2656
|
-
},
|
|
2657
|
-
banner: {
|
|
2658
|
-
js: "globalThis.process||={env:{},platform:'browser',versions:{},argv:[],cwd:()=>'/'};"
|
|
2867
|
+
multi,
|
|
2868
|
+
dashboards,
|
|
2869
|
+
dispose: async () => {
|
|
2870
|
+
await Promise.all(runners.map((r) => r.dispose().catch(() => {
|
|
2871
|
+
})));
|
|
2659
2872
|
}
|
|
2660
2873
|
};
|
|
2661
2874
|
}
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2875
|
+
function navTree(all) {
|
|
2876
|
+
const byDataset = /* @__PURE__ */ new Map();
|
|
2877
|
+
for (const d of all) byDataset.set(d.dataset, [...byDataset.get(d.dataset) ?? [], d]);
|
|
2878
|
+
return [...byDataset.entries()].map(([dataset, ds]) => ({
|
|
2879
|
+
dataset,
|
|
2880
|
+
title: ds[0]?.datasetLabel || void 0,
|
|
2881
|
+
dashboards: ds.map((d) => ({
|
|
2882
|
+
slug: d.slug,
|
|
2883
|
+
title: d.title || d.name,
|
|
2884
|
+
...d.description ? { description: d.description } : {}
|
|
2885
|
+
}))
|
|
2886
|
+
}));
|
|
2670
2887
|
}
|
|
2671
|
-
var rendersNoData = (d) => !d.query && (!d.tiles || d.tiles.length === 0);
|
|
2672
2888
|
|
|
2673
2889
|
// src/gather.ts
|
|
2674
|
-
var SKIP_DIRS = /* @__PURE__ */ new Set(["node_modules", ".git"]);
|
|
2675
2890
|
var DEVCONTAINER_PATH = ".devcontainer/devcontainer.json";
|
|
2676
2891
|
function gatherDirectory(dir) {
|
|
2677
2892
|
const files = [];
|
|
@@ -2765,6 +2980,40 @@ function gitInfo(dir) {
|
|
|
2765
2980
|
return {};
|
|
2766
2981
|
}
|
|
2767
2982
|
}
|
|
2983
|
+
function gitArchiveZip(dir) {
|
|
2984
|
+
const tmpIndex = join2(
|
|
2985
|
+
mkdtempSync(join2(tmpdir(), "malloyyo-index-")),
|
|
2986
|
+
"index"
|
|
2987
|
+
);
|
|
2988
|
+
const git = (args, encoding = "utf8") => execFileSync("git", args, {
|
|
2989
|
+
cwd: dir,
|
|
2990
|
+
env: { ...process.env, GIT_INDEX_FILE: tmpIndex },
|
|
2991
|
+
maxBuffer: 256 * 1024 * 1024,
|
|
2992
|
+
...encoding === "utf8" ? { encoding: "utf8" } : {},
|
|
2993
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
2994
|
+
});
|
|
2995
|
+
try {
|
|
2996
|
+
try {
|
|
2997
|
+
git(["rev-parse", "--is-inside-work-tree"]);
|
|
2998
|
+
} catch {
|
|
2999
|
+
throw new Error(
|
|
3000
|
+
`${dir} is not a git repository.
|
|
3001
|
+
A repo publishes what git tracks, so there has to be a git repo to ask. Run \`git init\` and commit, or publish from the repo's own directory.`
|
|
3002
|
+
);
|
|
3003
|
+
}
|
|
3004
|
+
try {
|
|
3005
|
+
git(["read-tree", "HEAD"]);
|
|
3006
|
+
} catch {
|
|
3007
|
+
}
|
|
3008
|
+
git(["add", "-A"]);
|
|
3009
|
+
const tree = git(["write-tree"]).trim();
|
|
3010
|
+
const fileCount = (git(["ls-files"]).trim().match(/\n/g)?.length ?? -1) + 1;
|
|
3011
|
+
const zip = git(["archive", "--format=zip", tree], "buffer");
|
|
3012
|
+
return { zip, fileCount };
|
|
3013
|
+
} finally {
|
|
3014
|
+
rmSync(dirname(tmpIndex), { recursive: true, force: true });
|
|
3015
|
+
}
|
|
3016
|
+
}
|
|
2768
3017
|
|
|
2769
3018
|
// src/lint.ts
|
|
2770
3019
|
import { existsSync as existsSync3, readdirSync as readdirSync2, readFileSync as readFileSync3 } from "node:fs";
|
|
@@ -2788,11 +3037,18 @@ function landingPageErrors(dir, file) {
|
|
|
2788
3037
|
return [`${file}: ${msg}`];
|
|
2789
3038
|
}
|
|
2790
3039
|
}
|
|
2791
|
-
async function lintDashboards(root) {
|
|
3040
|
+
async function lintDashboards(root, opts = {}) {
|
|
2792
3041
|
const abs = resolve(root);
|
|
2793
|
-
const runner = await makeRunner(abs);
|
|
3042
|
+
const runner = await makeRunner(abs, opts);
|
|
2794
3043
|
try {
|
|
2795
|
-
|
|
3044
|
+
const report = await runLint(abs, runner);
|
|
3045
|
+
if (!opts.datasetName) return report;
|
|
3046
|
+
let declared;
|
|
3047
|
+
try {
|
|
3048
|
+
declared = (await runner.datasetMeta()).title;
|
|
3049
|
+
} catch {
|
|
3050
|
+
}
|
|
3051
|
+
return { ...report, datasetTitle: datasetTitle(opts.datasetName, declared) };
|
|
2796
3052
|
} finally {
|
|
2797
3053
|
await runner.dispose();
|
|
2798
3054
|
}
|
|
@@ -2912,6 +3168,48 @@ function printLintReport(report) {
|
|
|
2912
3168
|
for (const w of d.warnings) console.log(` warning: ${w}`);
|
|
2913
3169
|
}
|
|
2914
3170
|
}
|
|
3171
|
+
var OLD_LAYOUT_NOTICE = [
|
|
3172
|
+
"This repo has index.malloy at its root \u2014 the old single-dataset layout.",
|
|
3173
|
+
"Repos now publish one dataset per directory under datasets/.",
|
|
3174
|
+
"",
|
|
3175
|
+
"It still works. To convert it, run `claude` here and ask it to convert this",
|
|
3176
|
+
'repo to the datasets/ layout, or read yo_help("repo/convert-single-dataset").'
|
|
3177
|
+
].join("\n");
|
|
3178
|
+
async function lintRepo(root) {
|
|
3179
|
+
const abs = resolve(root);
|
|
3180
|
+
const repoRoot = repoRootOf(abs);
|
|
3181
|
+
const layout = await layoutFromListing(fsLister(abs), abs);
|
|
3182
|
+
if (!layout.ok) {
|
|
3183
|
+
return { ok: false, layoutError: layout.error, datasets: [], ...layout.empty ? { empty: true } : {} };
|
|
3184
|
+
}
|
|
3185
|
+
const targets = layout.kind === "single" ? [{ name: "", dir: "" }] : layout.datasets.map((d) => ({ name: d.name, dir: d.dir }));
|
|
3186
|
+
const oldLayout = layout.kind === "single" && repoRoot === abs;
|
|
3187
|
+
const datasets = [];
|
|
3188
|
+
for (const t of targets) {
|
|
3189
|
+
const root2 = t.dir ? join3(abs, t.dir) : abs;
|
|
3190
|
+
const report = await lintDashboards(root2, {
|
|
3191
|
+
repoRoot,
|
|
3192
|
+
...t.name ? { datasetName: t.name } : {}
|
|
3193
|
+
});
|
|
3194
|
+
datasets.push({ name: t.name, dir: t.dir, title: report.datasetTitle ?? "", report });
|
|
3195
|
+
}
|
|
3196
|
+
return { ok: datasets.every((d) => d.report.ok), datasets, oldLayout };
|
|
3197
|
+
}
|
|
3198
|
+
function printRepoLintReport(repo) {
|
|
3199
|
+
if (repo.layoutError) {
|
|
3200
|
+
console.log(` \u2717 ${repo.layoutError}`);
|
|
3201
|
+
return;
|
|
3202
|
+
}
|
|
3203
|
+
const many = repo.datasets.length > 1;
|
|
3204
|
+
for (const d of repo.datasets) {
|
|
3205
|
+
if (many) {
|
|
3206
|
+
const asName = d.dir.split("/").pop() === d.name ? "" : ` \u2192 ${d.name}`;
|
|
3207
|
+
console.log(` ${d.dir}${asName} \u201C${d.title}\u201D`);
|
|
3208
|
+
}
|
|
3209
|
+
if (d.report.dashboards.length === 0 && many) console.log(" (no dashboards)");
|
|
3210
|
+
printLintReport(d.report);
|
|
3211
|
+
}
|
|
3212
|
+
}
|
|
2915
3213
|
|
|
2916
3214
|
// src/shared/env-refs.ts
|
|
2917
3215
|
function missingEnvRefs(configJson, env = process.env) {
|
|
@@ -2950,7 +3248,7 @@ import { spawn } from "node:child_process";
|
|
|
2950
3248
|
|
|
2951
3249
|
// src/store.ts
|
|
2952
3250
|
import { homedir } from "node:os";
|
|
2953
|
-
import { dirname, join as join4 } from "node:path";
|
|
3251
|
+
import { dirname as dirname2, join as join4 } from "node:path";
|
|
2954
3252
|
import { mkdirSync, readFileSync as readFileSync4, writeFileSync, existsSync as existsSync4, chmodSync } from "node:fs";
|
|
2955
3253
|
function credsPath() {
|
|
2956
3254
|
const base = process.env.XDG_CONFIG_HOME || join4(homedir(), ".config");
|
|
@@ -2970,7 +3268,7 @@ function loadCreds(url6) {
|
|
|
2970
3268
|
}
|
|
2971
3269
|
function saveCreds(url6, creds) {
|
|
2972
3270
|
const p = credsPath();
|
|
2973
|
-
mkdirSync(
|
|
3271
|
+
mkdirSync(dirname2(p), { recursive: true });
|
|
2974
3272
|
const all = readAll();
|
|
2975
3273
|
all[url6] = creds;
|
|
2976
3274
|
writeFileSync(p, JSON.stringify(all, null, 2) + "\n", { mode: 384 });
|
|
@@ -2988,7 +3286,7 @@ function clearCreds(url6) {
|
|
|
2988
3286
|
}
|
|
2989
3287
|
|
|
2990
3288
|
// package.json
|
|
2991
|
-
var version = "0.2.
|
|
3289
|
+
var version = "0.2.56";
|
|
2992
3290
|
|
|
2993
3291
|
// src/http.ts
|
|
2994
3292
|
var USER_AGENT = `malloyyo/${version}`;
|
|
@@ -3249,8 +3547,8 @@ Run: malloyyo login ${target.name}`);
|
|
|
3249
3547
|
}
|
|
3250
3548
|
|
|
3251
3549
|
// src/mcp.ts
|
|
3252
|
-
import
|
|
3253
|
-
import
|
|
3550
|
+
import fs5 from "node:fs";
|
|
3551
|
+
import path6 from "node:path";
|
|
3254
3552
|
import url4 from "node:url";
|
|
3255
3553
|
import { McpServer } from "@modelcontextprotocol/server";
|
|
3256
3554
|
import { StdioServerTransport } from "@modelcontextprotocol/server/stdio";
|
|
@@ -3329,7 +3627,7 @@ function defaultConfig(rootUrl) {
|
|
|
3329
3627
|
});
|
|
3330
3628
|
}
|
|
3331
3629
|
async function loadConfig2(root, reader) {
|
|
3332
|
-
const rootUrl = url4.pathToFileURL(root +
|
|
3630
|
+
const rootUrl = url4.pathToFileURL(root + path6.sep);
|
|
3333
3631
|
let discovered;
|
|
3334
3632
|
try {
|
|
3335
3633
|
discovered = await discoverConfig2(rootUrl, rootUrl, reader);
|
|
@@ -3351,13 +3649,13 @@ function fsReader2() {
|
|
|
3351
3649
|
if (u.protocol !== "file:") {
|
|
3352
3650
|
throw new Error(`unsupported URL scheme for import: ${u.href}`);
|
|
3353
3651
|
}
|
|
3354
|
-
return
|
|
3652
|
+
return fs5.promises.readFile(u, "utf8");
|
|
3355
3653
|
}
|
|
3356
3654
|
};
|
|
3357
3655
|
}
|
|
3358
3656
|
function resolveUnderRoot(root, p) {
|
|
3359
|
-
const abs = p.includes("://") ?
|
|
3360
|
-
if (abs !== root && !abs.startsWith(root +
|
|
3657
|
+
const abs = p.includes("://") ? path6.resolve(decodeURIComponent(new URL(p).pathname)) : path6.resolve(root, p);
|
|
3658
|
+
if (abs !== root && !abs.startsWith(root + path6.sep)) {
|
|
3361
3659
|
throw new Error(`path is outside the project root: ${p}`);
|
|
3362
3660
|
}
|
|
3363
3661
|
return abs;
|
|
@@ -3366,7 +3664,7 @@ function makeConfigSource(root) {
|
|
|
3366
3664
|
let cached;
|
|
3367
3665
|
const signature = () => ["malloy-config.json", "malloy-config-local.json"].map((name) => {
|
|
3368
3666
|
try {
|
|
3369
|
-
const st =
|
|
3667
|
+
const st = fs5.statSync(path6.join(root, name));
|
|
3370
3668
|
return `${name}:${st.mtimeMs}:${st.size}`;
|
|
3371
3669
|
} catch {
|
|
3372
3670
|
return `${name}:absent`;
|
|
@@ -3386,7 +3684,7 @@ function makeWithRuntime(root, currentConfig) {
|
|
|
3386
3684
|
return gateConfigProblems(problems, async () => {
|
|
3387
3685
|
const resolved = "url" in input ? { url: resolveUnderRoot(root, input.url) } : {
|
|
3388
3686
|
source: input.source,
|
|
3389
|
-
baseUrl: input.baseUrl ? resolveUnderRoot(root, input.baseUrl) : root +
|
|
3687
|
+
baseUrl: input.baseUrl ? resolveUnderRoot(root, input.baseUrl) : root + path6.sep
|
|
3390
3688
|
};
|
|
3391
3689
|
const { reader, entry, readSource } = prepareSource(fsReader2(), resolved);
|
|
3392
3690
|
const runtime = new Runtime2({ config, urlReader: reader });
|
|
@@ -3400,7 +3698,7 @@ function makeWithRuntime(root, currentConfig) {
|
|
|
3400
3698
|
}
|
|
3401
3699
|
function makeExploreHost(root, currentConfig) {
|
|
3402
3700
|
const withRuntime = makeWithRuntime(root, currentConfig);
|
|
3403
|
-
const published = (ref) => ref === ENTRY2 &&
|
|
3701
|
+
const published = (ref) => ref === ENTRY2 && fs5.existsSync(path6.join(root, ENTRY2));
|
|
3404
3702
|
return {
|
|
3405
3703
|
withModel: (ref, fn) => {
|
|
3406
3704
|
if (!published(ref)) throw new Error(`no published model '${ref}'`);
|
|
@@ -3421,7 +3719,7 @@ function makeDevelopHost(root, currentConfig) {
|
|
|
3421
3719
|
}
|
|
3422
3720
|
async function serveMcp(opts) {
|
|
3423
3721
|
await initConnections();
|
|
3424
|
-
const root =
|
|
3722
|
+
const root = path6.resolve(opts.root ?? process.cwd());
|
|
3425
3723
|
const mode = opts.mode ?? "explore";
|
|
3426
3724
|
const currentConfig = makeConfigSource(root);
|
|
3427
3725
|
const surface = mode === "develop" ? developSurface(makeDevelopHost(root, currentConfig)) : exploreSurface(makeExploreHost(root, currentConfig));
|
|
@@ -3444,8 +3742,8 @@ async function serveMcp(opts) {
|
|
|
3444
3742
|
|
|
3445
3743
|
// src/dashboard.ts
|
|
3446
3744
|
import http2 from "node:http";
|
|
3447
|
-
import
|
|
3448
|
-
import
|
|
3745
|
+
import fs6 from "node:fs";
|
|
3746
|
+
import path7 from "node:path";
|
|
3449
3747
|
import * as esbuild2 from "esbuild";
|
|
3450
3748
|
|
|
3451
3749
|
// src/shared/givens-url.ts
|
|
@@ -3474,25 +3772,53 @@ function safeJson(value) {
|
|
|
3474
3772
|
var esc = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
3475
3773
|
var NAV_CSS = `
|
|
3476
3774
|
.dash-nav{display:flex;gap:4px;align-items:center;padding:8px 14px;background:#000;font:13px system-ui,-apple-system,sans-serif;flex-wrap:wrap}
|
|
3477
|
-
/* The home icon
|
|
3478
|
-
keeps its own square hit area. */
|
|
3775
|
+
/* The home icon keeps its own square hit area. */
|
|
3479
3776
|
.dash-nav a.brand{display:inline-flex;align-items:center;justify-content:center;color:#9aa1ac;padding:5px;border-radius:6px;text-decoration:none}
|
|
3480
3777
|
.dash-nav a.brand:hover{background:#1f232a;color:#fff}
|
|
3481
3778
|
.dash-nav .brand svg{display:block}
|
|
3482
|
-
|
|
3483
|
-
|
|
3484
|
-
.
|
|
3485
|
-
.dash-
|
|
3779
|
+
/* The switcher: the same control the hosted app uses, in vanilla. A row of
|
|
3780
|
+
pills stops working at about five dashboards \u2014 a repo with four datasets has
|
|
3781
|
+
fifteen \u2014 so the list moves into a menu and the bar keeps one button. */
|
|
3782
|
+
.dash-pick{position:relative;font:13px system-ui,-apple-system,sans-serif}
|
|
3783
|
+
.dash-pick>button{display:flex;align-items:center;gap:6px;max-width:60vw;padding:4px 8px;border:0;border-radius:6px;
|
|
3784
|
+
background:transparent;color:#c9ced6;font:inherit;cursor:pointer}
|
|
3785
|
+
.dash-pick>button:hover{background:#1f232a;color:#fff}
|
|
3786
|
+
.dash-pick .ds{font-weight:600;color:#fff}
|
|
3787
|
+
.dash-pick .slash{color:#5b6270}
|
|
3788
|
+
.dash-pick .lbl{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
|
3789
|
+
.dash-pick .chev{flex:none;opacity:.6}
|
|
3790
|
+
.dash-pick[data-open="1"] .panel{display:block}
|
|
3791
|
+
.dash-pick .panel{display:none;position:absolute;left:0;top:100%;margin-top:4px;z-index:50;width:340px;max-width:92vw;
|
|
3792
|
+
background:#fff;color:#111;border:1px solid #d7dbe0;border-radius:8px;box-shadow:0 10px 30px rgba(0,0,0,.25)}
|
|
3793
|
+
.dash-pick .filter{padding:6px;border-bottom:1px solid #eceef1}
|
|
3794
|
+
.dash-pick .filter input{width:100%;box-sizing:border-box;padding:5px 8px;border:0;border-radius:4px;background:#f3f4f6;
|
|
3795
|
+
font:12px ui-monospace,SFMono-Regular,Menlo,monospace;color:#111}
|
|
3796
|
+
.dash-pick .filter input:focus{outline:0}
|
|
3797
|
+
.dash-pick .list{max-height:60vh;overflow-y:auto;padding:4px}
|
|
3798
|
+
.dash-pick .branch{display:flex;align-items:center;gap:5px;width:100%;padding:4px 6px;border:0;border-radius:4px;
|
|
3799
|
+
background:transparent;font:inherit;font-weight:600;color:#111;cursor:pointer;text-align:left}
|
|
3800
|
+
.dash-pick .branch:hover{background:#f3f4f6}
|
|
3801
|
+
.dash-pick .branch .tw{flex:none;transition:transform .12s}
|
|
3802
|
+
.dash-pick .branch[aria-expanded="true"] .tw{transform:rotate(90deg)}
|
|
3803
|
+
.dash-pick .leaf{display:block;padding:4px 8px;border-radius:4px;text-decoration:none;color:#374151;
|
|
3804
|
+
overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
|
3805
|
+
.dash-pick .leaf:hover{background:#f3f4f6}
|
|
3806
|
+
.dash-pick .leaf.on{background:#111;color:#fff}
|
|
3807
|
+
.dash-pick .kids{display:none}
|
|
3808
|
+
.dash-pick .kids.indent .leaf{margin-left:16px}
|
|
3809
|
+
.dash-pick .grp[data-open="1"] .kids{display:block}
|
|
3810
|
+
.dash-pick .empty{padding:6px 8px;color:#9aa1ac}
|
|
3811
|
+
@media (prefers-color-scheme:dark){
|
|
3812
|
+
.dash-pick .panel{background:#0b0d11;color:#e6e8eb;border-color:#262b33}
|
|
3813
|
+
.dash-pick .filter{border-bottom-color:#1a1e25}
|
|
3814
|
+
.dash-pick .filter input{background:#14181e;color:#e6e8eb}
|
|
3815
|
+
.dash-pick .branch{color:#e6e8eb}
|
|
3816
|
+
.dash-pick .branch:hover,.dash-pick .leaf:hover{background:#171b22}
|
|
3817
|
+
.dash-pick .leaf{color:#c9ced6}
|
|
3818
|
+
.dash-pick .leaf.on{background:#fff;color:#000}
|
|
3819
|
+
}
|
|
3486
3820
|
`;
|
|
3487
3821
|
var HOME_ICON = `<svg viewBox="0 0 24 24" width="17" height="17" fill="none" stroke="currentColor" stroke-width="1.9" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" focusable="false"><path d="M3 10.2 12 3.5l9 6.7"/><path d="M5.2 8.9V20h13.6V8.9"/><path d="M9.6 20v-6.2h4.8V20"/></svg>`;
|
|
3488
|
-
function navHtml(active, all, href, homeHref = "./") {
|
|
3489
|
-
const brand = `<a class="brand" href="${esc(homeHref)}" title="Home" aria-label="Home">${HOME_ICON}</a>`;
|
|
3490
|
-
if (all.length <= 1) return `<nav class="dash-nav">${brand}</nav>`;
|
|
3491
|
-
const links = all.map(
|
|
3492
|
-
(x) => `<a href="${esc(href(x.name))}"${x.name === active ? ' class="on"' : ""}>${esc(x.title || x.name)}</a>`
|
|
3493
|
-
).join("");
|
|
3494
|
-
return `<nav class="dash-nav">${brand}<span class="sep"></span>${links}</nav>`;
|
|
3495
|
-
}
|
|
3496
3822
|
function siblingList(current, all, href) {
|
|
3497
3823
|
return all.filter((d) => d.name !== current).map((d) => ({
|
|
3498
3824
|
name: d.name,
|
|
@@ -3501,21 +3827,93 @@ function siblingList(current, all, href) {
|
|
|
3501
3827
|
href: href(d.name)
|
|
3502
3828
|
}));
|
|
3503
3829
|
}
|
|
3830
|
+
function switcherHtml(activeSlug, tree, href, opts = {}) {
|
|
3831
|
+
const flat = tree.length <= 1;
|
|
3832
|
+
const groups = tree.map((ds) => {
|
|
3833
|
+
const open = flat || ds.dashboards.some((d) => d.slug === activeSlug) ? ' data-open="1"' : "";
|
|
3834
|
+
const leaves = ds.dashboards.map((d) => {
|
|
3835
|
+
const on = d.slug === activeSlug ? " on" : "";
|
|
3836
|
+
return `<a class="leaf${on}" href="${esc(href(d.slug))}"${d.description ? ` title="${esc(d.description)}"` : ""} data-find="${esc(`${ds.dataset} ${ds.title ?? ""} ${d.slug} ${d.title}`.toLowerCase())}">${esc(d.title)}</a>`;
|
|
3837
|
+
}).join("");
|
|
3838
|
+
if (flat) return `<div class="grp" data-open="1"><div class="kids">${leaves}</div></div>`;
|
|
3839
|
+
return `<div class="grp"${open} data-ds="${esc(`${ds.dataset} ${ds.title ?? ""}`.toLowerCase())}"><button class="branch" aria-expanded="${open ? "true" : "false"}"><svg class="tw" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M9 6l6 6-6 6"/></svg><span title="${esc(ds.dataset)}">${esc(ds.title ?? ds.dataset)}</span></button><div class="kids indent">${leaves}</div></div>`;
|
|
3840
|
+
}).join("");
|
|
3841
|
+
const filter = flat ? "" : `<div class="filter"><input type="search" placeholder="filter\u2026" aria-label="Filter dashboards"></div>`;
|
|
3842
|
+
return `<div class="dash-pick" data-open="0"><button type="button" aria-haspopup="menu" aria-expanded="false" title="All datasets and dashboards">` + (opts.dataset ? `<span class="ds">${esc(opts.dataset)}</span><span class="slash">/</span>` : "") + `<span class="lbl">${esc(opts.label ?? "dashboards")}</span><svg class="chev" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"/></svg></button><div class="panel" role="menu">${filter}<div class="list">${groups}` + (flat ? "" : `<div class="empty" hidden>no match</div>`) + `</div></div></div>`;
|
|
3843
|
+
}
|
|
3844
|
+
var SWITCHER_JS = `
|
|
3845
|
+
(function(){
|
|
3846
|
+
var p=document.querySelector('.dash-pick'); if(!p) return;
|
|
3847
|
+
var btn=p.querySelector(':scope>button'), inp=p.querySelector('.filter input');
|
|
3848
|
+
var grps=[].slice.call(p.querySelectorAll('.grp'));
|
|
3849
|
+
var none=p.querySelector('.empty');
|
|
3850
|
+
// How the page rendered: which branch holds the dashboard you are on. A
|
|
3851
|
+
// cleared search restores this rather than whatever the search expanded.
|
|
3852
|
+
var init=grps.map(function(g){return g.dataset.open==='1';});
|
|
3853
|
+
function setOpen(g,on){
|
|
3854
|
+
g.dataset.open=on?'1':'0';
|
|
3855
|
+
var b=g.querySelector('.branch'); if(b) b.setAttribute('aria-expanded',on?'true':'false');
|
|
3856
|
+
}
|
|
3857
|
+
function openState(on){
|
|
3858
|
+
p.dataset.open=on?'1':'0';
|
|
3859
|
+
btn.setAttribute('aria-expanded',on?'true':'false');
|
|
3860
|
+
if(!on&&inp){inp.value='';apply('');}
|
|
3861
|
+
if(on&&inp){inp.focus();}
|
|
3862
|
+
}
|
|
3863
|
+
function apply(q){
|
|
3864
|
+
q=q.trim().toLowerCase();
|
|
3865
|
+
var hits=0;
|
|
3866
|
+
grps.forEach(function(g,i){
|
|
3867
|
+
var ds=g.dataset.ds||'', dsHit=!!q&&ds.indexOf(q)>=0;
|
|
3868
|
+
var any=false;
|
|
3869
|
+
[].slice.call(g.querySelectorAll('.leaf')).forEach(function(a){
|
|
3870
|
+
var hit=!q||dsHit||(a.dataset.find||'').indexOf(q)>=0;
|
|
3871
|
+
a.style.display=hit?'':'none'; if(hit){any=true;hits++;}
|
|
3872
|
+
});
|
|
3873
|
+
g.style.display=any?'':'none';
|
|
3874
|
+
// While searching, everything that survived is open; with the box empty,
|
|
3875
|
+
// back to how the page rendered.
|
|
3876
|
+
setOpen(g,q?any:init[i]);
|
|
3877
|
+
});
|
|
3878
|
+
if(none) none.hidden=!q||hits>0;
|
|
3879
|
+
}
|
|
3880
|
+
btn.addEventListener('click',function(e){e.stopPropagation();openState(p.dataset.open!=='1');});
|
|
3881
|
+
grps.forEach(function(g){
|
|
3882
|
+
var b=g.querySelector('.branch'); if(!b) return;
|
|
3883
|
+
b.addEventListener('click',function(e){
|
|
3884
|
+
e.stopPropagation();
|
|
3885
|
+
setOpen(g,g.dataset.open!=='1');
|
|
3886
|
+
});
|
|
3887
|
+
});
|
|
3888
|
+
if(inp){inp.addEventListener('input',function(){apply(inp.value);});
|
|
3889
|
+
inp.addEventListener('click',function(e){e.stopPropagation();});}
|
|
3890
|
+
// Chrome inside the panel is not a dismissal \u2014 padding and the filter row's
|
|
3891
|
+
// margin used to bubble to the document handler and close the menu.
|
|
3892
|
+
var panel=p.querySelector('.panel');
|
|
3893
|
+
if(panel) panel.addEventListener('click',function(e){e.stopPropagation();});
|
|
3894
|
+
document.addEventListener('click',function(){ if(p.dataset.open==='1') openState(false); });
|
|
3895
|
+
document.addEventListener('keydown',function(e){ if(e.key==='Escape') openState(false); });
|
|
3896
|
+
})();
|
|
3897
|
+
`;
|
|
3898
|
+
function navWithSwitcher(activeSlug, tree, href, opts = {}) {
|
|
3899
|
+
const brand = `<a class="brand" href="${esc(opts.homeHref ?? "./")}" title="Home" aria-label="Home">${HOME_ICON}</a>`;
|
|
3900
|
+
return `<nav class="dash-nav">${brand}${switcherHtml(activeSlug, tree, href, opts)}</nav>`;
|
|
3901
|
+
}
|
|
3504
3902
|
|
|
3505
3903
|
// src/dashboard.ts
|
|
3506
|
-
var resolveFrameEntry = () =>
|
|
3507
|
-
var resolveInPageEntry = () =>
|
|
3904
|
+
var resolveFrameEntry = () => path7.join(resolveRuntimeDir(), "..", "frame-entry.tsx");
|
|
3905
|
+
var resolveInPageEntry = () => path7.join(resolveRuntimeDir(), "..", "frame-inpage-entry.tsx");
|
|
3508
3906
|
var inlineScript = (js) => js.replace(/<\/(script)/gi, String.raw`<\/$1`);
|
|
3509
3907
|
var esc2 = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
3510
3908
|
function makeBundler() {
|
|
3511
3909
|
const cache = /* @__PURE__ */ new Map();
|
|
3512
3910
|
const frameEntry = resolveFrameEntry();
|
|
3513
3911
|
const runtimeDir = resolveRuntimeDir();
|
|
3514
|
-
const runtimeIndex =
|
|
3515
|
-
const runtimeStamp = () =>
|
|
3912
|
+
const runtimeIndex = path7.join(runtimeDir, "index.ts");
|
|
3913
|
+
const runtimeStamp = () => fs6.statSync(frameEntry).mtimeMs + fs6.readdirSync(runtimeDir).map((f) => fs6.statSync(path7.join(runtimeDir, f)).mtimeMs).reduce((a, b) => a + b, 0);
|
|
3516
3914
|
return async function bundle(dash) {
|
|
3517
|
-
const stamp = runtimeStamp() + (dash.tsxPath ?
|
|
3518
|
-
const hit = cache.get(dash.
|
|
3915
|
+
const stamp = runtimeStamp() + (dash.tsxPath ? fs6.statSync(dash.tsxPath).mtimeMs : 0);
|
|
3916
|
+
const hit = cache.get(dash.slug);
|
|
3519
3917
|
if (hit && hit.stamp === stamp) return hit.js;
|
|
3520
3918
|
const result = await esbuild2.build({
|
|
3521
3919
|
entryPoints: [frameEntry],
|
|
@@ -3551,7 +3949,7 @@ function makeBundler() {
|
|
|
3551
3949
|
]
|
|
3552
3950
|
});
|
|
3553
3951
|
const js = result.outputFiles[0].text;
|
|
3554
|
-
cache.set(dash.
|
|
3952
|
+
cache.set(dash.slug, { stamp, js });
|
|
3555
3953
|
return js;
|
|
3556
3954
|
};
|
|
3557
3955
|
}
|
|
@@ -3559,7 +3957,7 @@ function makeInPageBundler() {
|
|
|
3559
3957
|
let cached;
|
|
3560
3958
|
const entry = resolveInPageEntry();
|
|
3561
3959
|
const runtimeDir = resolveRuntimeDir();
|
|
3562
|
-
const stampOf = () =>
|
|
3960
|
+
const stampOf = () => fs6.statSync(entry).mtimeMs + fs6.readdirSync(runtimeDir).map((f) => fs6.statSync(path7.join(runtimeDir, f)).mtimeMs).reduce((a, b) => a + b, 0);
|
|
3563
3961
|
return async function bundle() {
|
|
3564
3962
|
const stamp = stampOf();
|
|
3565
3963
|
if (cached && cached.stamp === stamp) return cached.js;
|
|
@@ -3580,7 +3978,7 @@ function makeInPageBundler() {
|
|
|
3580
3978
|
return js;
|
|
3581
3979
|
};
|
|
3582
3980
|
}
|
|
3583
|
-
var html = (body, title) => `<!doctype html><html><head><meta charset="utf-8"><title>${esc2(title)}</title><meta name="viewport" content="width=device-width,initial-scale=1"><style>${NAV_CSS}</style></head><body style="margin:0">${body}</body></html>`;
|
|
3981
|
+
var html = (body, title) => `<!doctype html><html><head><meta charset="utf-8"><title>${esc2(title)}</title><meta name="viewport" content="width=device-width,initial-scale=1"><style>${NAV_CSS}</style></head><body style="margin:0">${body}<script>${SWITCHER_JS}</script></body></html>`;
|
|
3584
3982
|
var dashLink = (n) => `?d=${encodeURIComponent(n)}`;
|
|
3585
3983
|
var DEV_PATHS = {
|
|
3586
3984
|
/** Sibling dashboard, for the switcher in the nav bar. */
|
|
@@ -3598,13 +3996,24 @@ var DEV_PATHS = {
|
|
|
3598
3996
|
/** The privileged query broker the parent shell brokers postMessage into. */
|
|
3599
3997
|
run: "api/run"
|
|
3600
3998
|
};
|
|
3601
|
-
var devSiblings = (dash, all) => siblingList(
|
|
3602
|
-
|
|
3603
|
-
|
|
3999
|
+
var devSiblings = (dash, all) => siblingList(
|
|
4000
|
+
dash.slug,
|
|
4001
|
+
all.map((d) => ({
|
|
4002
|
+
name: d.slug,
|
|
4003
|
+
title: d.title || d.name,
|
|
4004
|
+
...d.description ? { description: d.description } : {}
|
|
4005
|
+
})),
|
|
4006
|
+
dashLink
|
|
4007
|
+
);
|
|
4008
|
+
function navHtml(dash, all) {
|
|
4009
|
+
return navWithSwitcher(dash.slug, navTree(all), dashLink, {
|
|
4010
|
+
dataset: dash.datasetLabel || dash.dataset || void 0,
|
|
4011
|
+
label: dash.title || dash.name
|
|
4012
|
+
});
|
|
3604
4013
|
}
|
|
3605
4014
|
function inPageShell(dash, all, givenSpecs, initialGivens, initialUrlState, tileSpecs) {
|
|
3606
4015
|
const info = {
|
|
3607
|
-
name: dash.
|
|
4016
|
+
name: dash.slug,
|
|
3608
4017
|
query: dash.query,
|
|
3609
4018
|
title: dash.title,
|
|
3610
4019
|
description: dash.description,
|
|
@@ -3615,18 +4024,18 @@ function inPageShell(dash, all, givenSpecs, initialGivens, initialUrlState, tile
|
|
|
3615
4024
|
autorun: dash.autorun
|
|
3616
4025
|
};
|
|
3617
4026
|
return html(
|
|
3618
|
-
|
|
4027
|
+
navHtml(dash, all) + `<div id="root"></div><script>window.__DASHBOARD__=${safeJson(info)};window.__DASHBOARDS__=${safeJson(devSiblings(dash, all))};window.__GIVENS__=${safeJson(givenSpecs)};window.__INITIAL_GIVENS__=${safeJson(initialGivens)};window.__INITIAL_URLSTATE__=${safeJson(initialUrlState)}</script><script>try{new EventSource('${DEV_PATHS.events}').onmessage=()=>location.reload();}catch(e){}</script><script src="${DEV_PATHS.inPage(dash.slug)}"></script>`,
|
|
3619
4028
|
dash.title
|
|
3620
4029
|
);
|
|
3621
4030
|
}
|
|
3622
4031
|
function parentShell(dash, all, initialGivens, initialUrlState) {
|
|
3623
4032
|
const givensQs = Object.entries({ ...initialGivens, ...initialUrlState }).map(([k, v]) => `&${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("");
|
|
3624
|
-
const d = safeJson(dash.
|
|
3625
|
-
const nav =
|
|
4033
|
+
const d = safeJson(dash.slug);
|
|
4034
|
+
const nav = navHtml(dash, all);
|
|
3626
4035
|
return html(
|
|
3627
4036
|
`<div style="display:flex;flex-direction:column;height:100vh">` + nav + // allow-popups(+escape-sandbox): let a # link mark open its target in a
|
|
3628
4037
|
// normal new tab on click instead of being blocked by the sandbox.
|
|
3629
|
-
`<iframe id="f" sandbox="allow-scripts allow-popups allow-popups-to-escape-sandbox" src="${DEV_PATHS.frame(dash.
|
|
4038
|
+
`<iframe id="f" sandbox="allow-scripts allow-popups allow-popups-to-escape-sandbox" src="${DEV_PATHS.frame(dash.slug)}${givensQs}" style="border:0;flex:1;width:100%"></iframe></div><script>
|
|
3630
4039
|
const f=document.getElementById('f');
|
|
3631
4040
|
try{new EventSource('${DEV_PATHS.events}').onmessage=()=>location.reload();}catch(e){}
|
|
3632
4041
|
// The shareable URL has TWO namespaces the frame syncs independently:
|
|
@@ -3683,7 +4092,7 @@ function urlStateFromUrl(url6) {
|
|
|
3683
4092
|
}
|
|
3684
4093
|
function frameDoc(dash, all, givenSpecs, initialGivens, initialUrlState, bundleJs, tileSpecs) {
|
|
3685
4094
|
const info = {
|
|
3686
|
-
name: dash.
|
|
4095
|
+
name: dash.slug,
|
|
3687
4096
|
query: dash.query,
|
|
3688
4097
|
title: dash.title,
|
|
3689
4098
|
description: dash.description,
|
|
@@ -3707,29 +4116,26 @@ async function readBody(req) {
|
|
|
3707
4116
|
}
|
|
3708
4117
|
async function serveDashboard(opts) {
|
|
3709
4118
|
await initConnections();
|
|
3710
|
-
const root =
|
|
4119
|
+
const root = path7.resolve(opts.root ?? process.cwd());
|
|
3711
4120
|
const port = opts.port ?? 4173;
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
throw new Error(`No index.malloy at ${root} \u2014 run this from a Malloy model repo.`);
|
|
3715
|
-
}
|
|
3716
|
-
let dashboards = await discoverDashboards(root, runner);
|
|
4121
|
+
let repo = await discoverRepoDashboards(root);
|
|
4122
|
+
let dashboards = repo.dashboards;
|
|
3717
4123
|
if (dashboards.length === 0) {
|
|
3718
4124
|
throw new Error(
|
|
3719
4125
|
`No dashboards declared \u2014 tag a top-level query with \`# artifact title="\u2026"\` in the model.`
|
|
3720
4126
|
);
|
|
3721
4127
|
}
|
|
3722
|
-
let
|
|
4128
|
+
let bySlug = new Map(dashboards.map((d) => [d.slug, d]));
|
|
3723
4129
|
const bundle = makeBundler();
|
|
3724
4130
|
const inPageBundle = makeInPageBundler();
|
|
3725
|
-
const pick = (url6) =>
|
|
4131
|
+
const pick = (url6) => bySlug.get(url6.searchParams.get("d") ?? dashboards[0].slug) ?? dashboards[0];
|
|
3726
4132
|
async function resolveGivens(dash) {
|
|
3727
4133
|
if (rendersNoData(dash)) return { ok: true, union: [] };
|
|
3728
4134
|
if (dash.tiles && dash.entryFile) {
|
|
3729
|
-
const t = await runner.dashboardTiles(dash.entryFile, dash.tiles);
|
|
4135
|
+
const t = await dash.runner.dashboardTiles(dash.entryFile, dash.tiles);
|
|
3730
4136
|
return { ok: true, union: t.union, tiles: t.tiles };
|
|
3731
4137
|
}
|
|
3732
|
-
const specs = dash.entryFile ? await runner.givensForQueryIn(dash.entryFile, dash.query) : await runner.givensForQuery(dash.query);
|
|
4138
|
+
const specs = dash.entryFile ? await dash.runner.givensForQueryIn(dash.entryFile, dash.query) : await dash.runner.givensForQuery(dash.query);
|
|
3733
4139
|
if (!specs.ok) return { ok: false, error: specs.error };
|
|
3734
4140
|
return { ok: true, union: specs.givens };
|
|
3735
4141
|
}
|
|
@@ -3739,14 +4145,18 @@ async function serveDashboard(opts) {
|
|
|
3739
4145
|
};
|
|
3740
4146
|
let debounce;
|
|
3741
4147
|
try {
|
|
3742
|
-
|
|
4148
|
+
fs6.watch(root, { recursive: true }, (_evt, filename) => {
|
|
3743
4149
|
const f = filename?.toString() ?? "";
|
|
3744
4150
|
if (!f.endsWith(".malloy") && !f.includes("dashboards")) return;
|
|
3745
4151
|
clearTimeout(debounce);
|
|
3746
4152
|
debounce = setTimeout(() => {
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
4153
|
+
discoverRepoDashboards(root).then(async (next) => {
|
|
4154
|
+
const old = repo;
|
|
4155
|
+
repo = next;
|
|
4156
|
+
dashboards = next.dashboards;
|
|
4157
|
+
bySlug = new Map(dashboards.map((d) => [d.slug, d]));
|
|
4158
|
+
await old.dispose().catch(() => {
|
|
4159
|
+
});
|
|
3750
4160
|
console.error(` \u21BB ${f} changed \u2014 reloading`);
|
|
3751
4161
|
notifyReload();
|
|
3752
4162
|
}).catch((e) => console.error(` ! model error after ${f} changed: ${e.message}`));
|
|
@@ -3832,10 +4242,10 @@ async function serveDashboard(opts) {
|
|
|
3832
4242
|
}
|
|
3833
4243
|
if (url6.pathname === "/api/run" && req.method === "POST") {
|
|
3834
4244
|
const { d, query, malloy, givens } = JSON.parse(await readBody(req));
|
|
3835
|
-
const dash =
|
|
4245
|
+
const dash = bySlug.get(d);
|
|
3836
4246
|
if (!dash) return send(404, "application/json", JSON.stringify({ ok: false, problems: [{ message: `no dashboard '${d}'` }] }));
|
|
3837
4247
|
const entry = dash.entryFile;
|
|
3838
|
-
const out = typeof malloy === "string" ? entry ? await runner.runTextIn(entry, malloy, givens ?? {}) : await runner.runText(malloy, givens ?? {}) : entry ? await runner.runIn(entry, String(query ?? ""), givens ?? {}) : await runner.run(String(query ?? dash.query), givens ?? {});
|
|
4248
|
+
const out = typeof malloy === "string" ? entry ? await dash.runner.runTextIn(entry, malloy, givens ?? {}) : await dash.runner.runText(malloy, givens ?? {}) : entry ? await dash.runner.runIn(entry, String(query ?? ""), givens ?? {}) : await dash.runner.run(String(query ?? dash.query), givens ?? {});
|
|
3839
4249
|
return send(200, "application/json", JSON.stringify(out));
|
|
3840
4250
|
}
|
|
3841
4251
|
send(404, "text/plain", "not found");
|
|
@@ -3850,7 +4260,9 @@ async function serveDashboard(opts) {
|
|
|
3850
4260
|
console.error(` http://localhost:${port}/`);
|
|
3851
4261
|
for (const d of dashboards) {
|
|
3852
4262
|
const kind = d.tsxPath ? "custom (iframe)" : "tag-only (in-page)";
|
|
3853
|
-
console.error(
|
|
4263
|
+
console.error(
|
|
4264
|
+
` \u2022 ${d.slug} (${kind}) \u2192 http://localhost:${port}/?d=${encodeURIComponent(d.slug)}`
|
|
4265
|
+
);
|
|
3854
4266
|
}
|
|
3855
4267
|
console.error(` Ctrl-C to stop.
|
|
3856
4268
|
`);
|
|
@@ -3859,15 +4271,15 @@ async function serveDashboard(opts) {
|
|
|
3859
4271
|
}
|
|
3860
4272
|
|
|
3861
4273
|
// src/bundle.ts
|
|
3862
|
-
import
|
|
3863
|
-
import
|
|
4274
|
+
import fs8 from "node:fs";
|
|
4275
|
+
import path9 from "node:path";
|
|
3864
4276
|
import { createRequire as createRequire3 } from "node:module";
|
|
3865
4277
|
import * as esbuild3 from "esbuild";
|
|
3866
4278
|
|
|
3867
4279
|
// src/static-server.ts
|
|
3868
|
-
import
|
|
4280
|
+
import fs7 from "node:fs";
|
|
3869
4281
|
import http3 from "node:http";
|
|
3870
|
-
import
|
|
4282
|
+
import path8 from "node:path";
|
|
3871
4283
|
var MIME = {
|
|
3872
4284
|
".html": "text/html; charset=utf-8",
|
|
3873
4285
|
".js": "text/javascript; charset=utf-8",
|
|
@@ -3882,12 +4294,12 @@ var MIME = {
|
|
|
3882
4294
|
function serveStatic(dir, port) {
|
|
3883
4295
|
const server = http3.createServer((req, res) => {
|
|
3884
4296
|
const rel = decodeURIComponent((req.url ?? "/").split("?")[0]);
|
|
3885
|
-
let file =
|
|
4297
|
+
let file = path8.join(dir, rel === "/" ? "index.html" : rel);
|
|
3886
4298
|
if (!file.startsWith(dir)) return void res.writeHead(403).end();
|
|
3887
|
-
if (
|
|
3888
|
-
if (!
|
|
3889
|
-
const st =
|
|
3890
|
-
const type = MIME[
|
|
4299
|
+
if (fs7.existsSync(file) && fs7.statSync(file).isDirectory()) file = path8.join(file, "index.html");
|
|
4300
|
+
if (!fs7.existsSync(file)) return void res.writeHead(404).end("not found");
|
|
4301
|
+
const st = fs7.statSync(file);
|
|
4302
|
+
const type = MIME[path8.extname(file)] ?? "application/octet-stream";
|
|
3891
4303
|
const range = req.headers.range;
|
|
3892
4304
|
if (range) {
|
|
3893
4305
|
const m = /bytes=(\d*)-(\d*)/.exec(range);
|
|
@@ -3899,10 +4311,10 @@ function serveStatic(dir, port) {
|
|
|
3899
4311
|
"Accept-Ranges": "bytes",
|
|
3900
4312
|
"Content-Length": end - start + 1
|
|
3901
4313
|
});
|
|
3902
|
-
return void
|
|
4314
|
+
return void fs7.createReadStream(file, { start, end }).pipe(res);
|
|
3903
4315
|
}
|
|
3904
4316
|
res.writeHead(200, { "Content-Type": type, "Content-Length": st.size, "Accept-Ranges": "bytes" });
|
|
3905
|
-
|
|
4317
|
+
fs7.createReadStream(file).pipe(res);
|
|
3906
4318
|
});
|
|
3907
4319
|
return new Promise((resolve5, reject) => {
|
|
3908
4320
|
let attempt = 0;
|
|
@@ -3919,7 +4331,7 @@ function serveStatic(dir, port) {
|
|
|
3919
4331
|
if (p !== port) console.log(`
|
|
3920
4332
|
(port ${port} busy \u2014 using ${p})`);
|
|
3921
4333
|
console.log(`
|
|
3922
|
-
serving ${
|
|
4334
|
+
serving ${path8.basename(dir)}/ on http://localhost:${p} (ctrl-c to stop)`);
|
|
3923
4335
|
resolve5();
|
|
3924
4336
|
});
|
|
3925
4337
|
};
|
|
@@ -3932,15 +4344,15 @@ var require3 = createRequire3(import.meta.url);
|
|
|
3932
4344
|
var esc3 = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
3933
4345
|
function inlineModelFiles(root) {
|
|
3934
4346
|
const files = {};
|
|
3935
|
-
const skip = /* @__PURE__ */ new Set(["node_modules", ".git", "docs", "dist"]);
|
|
3936
4347
|
const walk = (dir) => {
|
|
3937
|
-
for (const entry of
|
|
3938
|
-
if (entry.name.startsWith(".") ||
|
|
3939
|
-
const abs =
|
|
3940
|
-
if (entry.isDirectory())
|
|
3941
|
-
|
|
3942
|
-
|
|
3943
|
-
|
|
4348
|
+
for (const entry of fs8.readdirSync(dir, { withFileTypes: true })) {
|
|
4349
|
+
if (entry.name.startsWith(".") || SKIP_DIRS.has(entry.name)) continue;
|
|
4350
|
+
const abs = path9.join(dir, entry.name);
|
|
4351
|
+
if (entry.isDirectory()) {
|
|
4352
|
+
if (!isBundleOutput(abs)) walk(abs);
|
|
4353
|
+
} else if (entry.name.endsWith(".malloy")) {
|
|
4354
|
+
const rel = path9.relative(root, abs).split(path9.sep).join("/");
|
|
4355
|
+
files[`file:///${rel}`] = fs8.readFileSync(abs, "utf8");
|
|
3944
4356
|
}
|
|
3945
4357
|
}
|
|
3946
4358
|
};
|
|
@@ -4003,22 +4415,23 @@ function copyDuckDBAssets(outDir) {
|
|
|
4003
4415
|
"duckdb-browser-mvp.worker.js",
|
|
4004
4416
|
"duckdb-browser-eh.worker.js"
|
|
4005
4417
|
];
|
|
4006
|
-
const dir =
|
|
4007
|
-
|
|
4418
|
+
const dir = path9.join(outDir, "duckdb");
|
|
4419
|
+
fs8.mkdirSync(dir, { recursive: true });
|
|
4008
4420
|
const copied = [];
|
|
4009
4421
|
for (const n of names) {
|
|
4010
4422
|
const src = require3.resolve(`@duckdb/duckdb-wasm/dist/${n}`);
|
|
4011
|
-
|
|
4423
|
+
fs8.copyFileSync(src, path9.join(dir, n));
|
|
4012
4424
|
copied.push(n);
|
|
4013
4425
|
}
|
|
4014
4426
|
return copied;
|
|
4015
4427
|
}
|
|
4428
|
+
var pageFile = (slug) => slug.replace(/\//g, ".");
|
|
4429
|
+
var pageLink = (cleanUrls) => (slug) => cleanUrls ? `./${encodeURIComponent(pageFile(slug))}` : `./${encodeURIComponent(pageFile(slug))}.html`;
|
|
4016
4430
|
function navFor(dash, all, cleanUrls) {
|
|
4017
|
-
return
|
|
4018
|
-
dash.
|
|
4019
|
-
|
|
4020
|
-
|
|
4021
|
-
);
|
|
4431
|
+
return navWithSwitcher(dash.slug, navTree(all), pageLink(cleanUrls), {
|
|
4432
|
+
dataset: dash.datasetLabel || dash.dataset || void 0,
|
|
4433
|
+
label: dash.title || dash.name
|
|
4434
|
+
});
|
|
4022
4435
|
}
|
|
4023
4436
|
function analyticsSnippet(id) {
|
|
4024
4437
|
if (!id) return "";
|
|
@@ -4029,11 +4442,15 @@ gtag('js',new Date());gtag('config',${j});</script>`;
|
|
|
4029
4442
|
}
|
|
4030
4443
|
function page(dash, all, title, givenSpecs, tileSpecs, cleanUrls, analytics) {
|
|
4031
4444
|
const info = {
|
|
4032
|
-
|
|
4445
|
+
// How this page is addressed ON THIS HOST, which is what the runtime pushes
|
|
4446
|
+
// into the URL. A bundle's pages are flat files, so it is the file id — the
|
|
4447
|
+
// dev server uses `?d=<slug>`, where a slash is fine.
|
|
4448
|
+
name: pageFile(dash.slug),
|
|
4033
4449
|
query: dash.query,
|
|
4034
4450
|
title: dash.title,
|
|
4035
4451
|
description: dash.description,
|
|
4036
|
-
|
|
4452
|
+
// Repo-relative — the inlined file map is keyed that way.
|
|
4453
|
+
entryFile: dash.repoEntryFile ?? dash.entryFile,
|
|
4037
4454
|
tiles: dash.tiles,
|
|
4038
4455
|
tileSpecs,
|
|
4039
4456
|
dashboard_columns: dash.dashboard_columns,
|
|
@@ -4051,6 +4468,7 @@ ${analyticsSnippet(analytics)}
|
|
|
4051
4468
|
</head>
|
|
4052
4469
|
<body>
|
|
4053
4470
|
${navFor(dash, all, cleanUrls)}
|
|
4471
|
+
<script>${SWITCHER_JS}</script>
|
|
4054
4472
|
<div id="root"></div>
|
|
4055
4473
|
<script>
|
|
4056
4474
|
window.__DASHBOARD__ = ${safeJson(info)};
|
|
@@ -4065,20 +4483,21 @@ window.__GIVENS__ = ${safeJson(givenSpecs)};
|
|
|
4065
4483
|
// the runtime keys off, so every shareable link fell back to defaults).
|
|
4066
4484
|
</script>
|
|
4067
4485
|
<script src="./assets/model-files.js"></script>
|
|
4068
|
-
<script type="module" src="./assets/${dash.
|
|
4486
|
+
<script type="module" src="./assets/${pageFile(dash.slug)}.js"></script>
|
|
4069
4487
|
</body>
|
|
4070
4488
|
</html>
|
|
4071
4489
|
`;
|
|
4072
4490
|
}
|
|
4073
4491
|
function indexPage(dashboards, title, custom, cleanUrls, analytics) {
|
|
4074
|
-
const link = (
|
|
4492
|
+
const link = pageLink(cleanUrls);
|
|
4493
|
+
const multi = new Set(dashboards.map((d) => d.dataset)).size > 1;
|
|
4075
4494
|
const listed = dashboards.filter((d) => !rendersNoData(d));
|
|
4076
4495
|
const body = custom ? `<div id="root"></div>
|
|
4077
4496
|
<script>window.__DASHBOARDS__ = ${safeJson(
|
|
4078
|
-
listed.map((d) => ({ name: d.
|
|
4497
|
+
listed.map((d) => ({ name: d.slug, title: d.title, description: d.description, href: link(d.slug) }))
|
|
4079
4498
|
)};</script>
|
|
4080
4499
|
<script type="module" src="./assets/index.js"></script>` : `<main class="index"><h1>${esc3(title)}</h1><ul>` + listed.map(
|
|
4081
|
-
(d) => `<li><a href="${link(d.
|
|
4500
|
+
(d) => `<li><a href="${link(d.slug)}">` + (multi && d.datasetLabel ? `<em>${esc3(d.datasetLabel)}</em>` : "") + `<strong>${esc3(d.title || d.name)}</strong>` + (d.description ? `<span>${esc3(d.description)}</span>` : "") + `</a></li>`
|
|
4082
4501
|
).join("") + `</ul></main>`;
|
|
4083
4502
|
return `<!doctype html>
|
|
4084
4503
|
<html lang="en">
|
|
@@ -4090,8 +4509,9 @@ function indexPage(dashboards, title, custom, cleanUrls, analytics) {
|
|
|
4090
4509
|
${analyticsSnippet(analytics)}
|
|
4091
4510
|
</head>
|
|
4092
4511
|
<body>
|
|
4093
|
-
${
|
|
4512
|
+
${navWithSwitcher("", navTree(dashboards), link)}
|
|
4094
4513
|
${body}
|
|
4514
|
+
<script>${SWITCHER_JS}</script>
|
|
4095
4515
|
</body>
|
|
4096
4516
|
</html>
|
|
4097
4517
|
`;
|
|
@@ -4103,42 +4523,47 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 -apple-system,B
|
|
|
4103
4523
|
.index{max-width:760px;margin:0 auto;padding:48px 20px}
|
|
4104
4524
|
.index h1{font-size:22px;font-weight:650;margin:0 0 22px}
|
|
4105
4525
|
.index ul{list-style:none;padding:0;margin:0;display:grid;gap:10px}
|
|
4526
|
+
.index li a em{display:block;font-style:normal;font-size:12px;color:var(--muted);margin-bottom:2px}
|
|
4106
4527
|
.index a{display:flex;flex-direction:column;gap:3px;padding:16px 18px;border:1px solid var(--line);border-radius:12px;background:var(--card);text-decoration:none;color:inherit}
|
|
4107
4528
|
.index a:hover{border-color:var(--accent)}
|
|
4108
4529
|
.index span{font-size:13px;color:var(--muted)}
|
|
4109
4530
|
` + NAV_CSS;
|
|
4110
4531
|
async function bundleDashboards(opts = {}) {
|
|
4111
|
-
const root =
|
|
4112
|
-
const outDir =
|
|
4113
|
-
const title = opts.title ??
|
|
4532
|
+
const root = path9.resolve(opts.root ?? process.cwd());
|
|
4533
|
+
const outDir = path9.resolve(root, opts.out ?? "docs");
|
|
4534
|
+
const title = opts.title ?? path9.basename(root);
|
|
4114
4535
|
const target = opts.target ?? "pages";
|
|
4115
4536
|
const analytics = opts.analytics ?? readSiteConfig(root).analytics;
|
|
4116
4537
|
const cleanUrls = target === "vercel";
|
|
4117
4538
|
const selfHostDuckdb = opts.duckdb === "bundled";
|
|
4118
|
-
const
|
|
4119
|
-
const dashboards =
|
|
4120
|
-
if (dashboards.length === 0)
|
|
4121
|
-
|
|
4539
|
+
const repo = await discoverRepoDashboards(root);
|
|
4540
|
+
const dashboards = repo.dashboards;
|
|
4541
|
+
if (dashboards.length === 0) {
|
|
4542
|
+
throw new Error(
|
|
4543
|
+
repo.multi ? `no dashboards found \u2014 each dataset keeps its own in datasets/<name>/dashboards/` : `no dashboards found in ${path9.join(root, "dashboards")}`
|
|
4544
|
+
);
|
|
4545
|
+
}
|
|
4546
|
+
const manifestPath = path9.join(outDir, ".bundle-manifest.json");
|
|
4122
4547
|
let priorData = [];
|
|
4123
4548
|
try {
|
|
4124
|
-
priorData = JSON.parse(
|
|
4549
|
+
priorData = JSON.parse(fs8.readFileSync(manifestPath, "utf8")).dataFiles ?? [];
|
|
4125
4550
|
} catch {
|
|
4126
4551
|
}
|
|
4127
4552
|
for (const sub of ["assets", "duckdb"]) {
|
|
4128
|
-
|
|
4553
|
+
fs8.rmSync(path9.join(outDir, sub), { recursive: true, force: true });
|
|
4129
4554
|
}
|
|
4130
|
-
if (
|
|
4131
|
-
for (const f of
|
|
4132
|
-
if (f.endsWith(".html"))
|
|
4555
|
+
if (fs8.existsSync(outDir)) {
|
|
4556
|
+
for (const f of fs8.readdirSync(outDir)) {
|
|
4557
|
+
if (f.endsWith(".html")) fs8.rmSync(path9.join(outDir, f), { force: true });
|
|
4133
4558
|
}
|
|
4134
4559
|
}
|
|
4135
|
-
|
|
4560
|
+
fs8.mkdirSync(path9.join(outDir, "assets"), { recursive: true });
|
|
4136
4561
|
if (target === "pages") {
|
|
4137
|
-
|
|
4562
|
+
fs8.writeFileSync(path9.join(outDir, ".nojekyll"), "");
|
|
4138
4563
|
} else {
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4564
|
+
fs8.rmSync(path9.join(outDir, ".nojekyll"), { force: true });
|
|
4565
|
+
fs8.writeFileSync(
|
|
4566
|
+
path9.join(outDir, "vercel.json"),
|
|
4142
4567
|
JSON.stringify(
|
|
4143
4568
|
{
|
|
4144
4569
|
$schema: "https://openapi.vercel.sh/vercel.json",
|
|
@@ -4160,53 +4585,53 @@ async function bundleDashboards(opts = {}) {
|
|
|
4160
4585
|
);
|
|
4161
4586
|
}
|
|
4162
4587
|
const modelFiles = inlineModelFiles(root);
|
|
4163
|
-
const outRel =
|
|
4588
|
+
const outRel = path9.relative(root, outDir).split(path9.sep).join("/");
|
|
4164
4589
|
const usedFiles = reachableModelFiles(
|
|
4165
4590
|
modelFiles,
|
|
4166
|
-
dashboards.map((d) => d.entryFile).filter((f) => !!f)
|
|
4591
|
+
dashboards.map((d) => d.repoEntryFile ?? d.entryFile).filter((f) => !!f)
|
|
4167
4592
|
);
|
|
4168
4593
|
const { map: tableFiles, copies } = tableFilePlan(usedFiles, outRel);
|
|
4169
4594
|
for (const rel of copies) {
|
|
4170
|
-
const from =
|
|
4171
|
-
if (!
|
|
4595
|
+
const from = path9.join(root, rel);
|
|
4596
|
+
if (!fs8.existsSync(from)) {
|
|
4172
4597
|
throw new Error(
|
|
4173
4598
|
`model reads '${rel}' but ${from} does not exist.
|
|
4174
4599
|
Data files are referenced by a path relative to the project root.`
|
|
4175
4600
|
);
|
|
4176
4601
|
}
|
|
4177
|
-
const to =
|
|
4178
|
-
|
|
4179
|
-
|
|
4602
|
+
const to = path9.join(outDir, rel);
|
|
4603
|
+
fs8.mkdirSync(path9.dirname(to), { recursive: true });
|
|
4604
|
+
fs8.copyFileSync(from, to);
|
|
4180
4605
|
}
|
|
4181
4606
|
const copiedData = copies;
|
|
4182
4607
|
for (const stale of priorData) {
|
|
4183
4608
|
if (copies.includes(stale)) continue;
|
|
4184
|
-
|
|
4609
|
+
fs8.rmSync(path9.join(outDir, stale), { force: true });
|
|
4185
4610
|
try {
|
|
4186
|
-
|
|
4611
|
+
fs8.rmdirSync(path9.dirname(path9.join(outDir, stale)));
|
|
4187
4612
|
} catch {
|
|
4188
4613
|
}
|
|
4189
4614
|
console.log(` removed stale ${stale}`);
|
|
4190
4615
|
}
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
|
|
4616
|
+
fs8.writeFileSync(manifestPath, JSON.stringify({ dataFiles: copies }, null, 2) + "\n");
|
|
4617
|
+
fs8.writeFileSync(
|
|
4618
|
+
path9.join(outDir, "assets", "model-files.js"),
|
|
4194
4619
|
`window.__MODEL_FILES__ = ${JSON.stringify(modelFiles)};
|
|
4195
4620
|
window.__TABLE_FILES__ = ${JSON.stringify(tableFiles)};
|
|
4196
4621
|
` + (selfHostDuckdb ? `window.__DUCKDB_BASE__ = "./duckdb/";
|
|
4197
4622
|
` : "")
|
|
4198
4623
|
);
|
|
4199
|
-
const landing = ["jsx", "tsx"].map((ext) =>
|
|
4624
|
+
const landing = ["jsx", "tsx"].map((ext) => path9.join(root, "dashboards", `index.${ext}`)).find((f) => fs8.existsSync(f));
|
|
4200
4625
|
const runtimeDir = resolveRuntimeDir();
|
|
4201
|
-
const runtimeIndex =
|
|
4202
|
-
const wasmEntry =
|
|
4203
|
-
const byEntry = new Map(dashboards.map((d) => [`vdash:${d.
|
|
4626
|
+
const runtimeIndex = path9.join(runtimeDir, "index.ts");
|
|
4627
|
+
const wasmEntry = path9.join(runtimeDir, "..", "frame-wasm-entry.tsx");
|
|
4628
|
+
const byEntry = new Map(dashboards.map((d) => [`vdash:${pageFile(d.slug)}`, d]));
|
|
4204
4629
|
await esbuild3.build({
|
|
4205
|
-
entryPoints: Object.fromEntries(dashboards.map((d) => [d.
|
|
4630
|
+
entryPoints: Object.fromEntries(dashboards.map((d) => [pageFile(d.slug), `vdash:${pageFile(d.slug)}`])),
|
|
4206
4631
|
bundle: true,
|
|
4207
4632
|
splitting: true,
|
|
4208
4633
|
format: "esm",
|
|
4209
|
-
outdir:
|
|
4634
|
+
outdir: path9.join(outDir, "assets"),
|
|
4210
4635
|
minify: true,
|
|
4211
4636
|
logLevel: "warning",
|
|
4212
4637
|
...browserBuildBase(),
|
|
@@ -4226,8 +4651,11 @@ boot(Dashboard);
|
|
|
4226
4651
|
`,
|
|
4227
4652
|
loader: "js",
|
|
4228
4653
|
// Resolve the component's own imports (react, @malloyyo/dashboard)
|
|
4229
|
-
// from the
|
|
4230
|
-
|
|
4654
|
+
// from the DATASET's directory, matching `dashboard dev`. Not the
|
|
4655
|
+
// repo root: a multi-dataset repo has no `dashboards/` of its own,
|
|
4656
|
+
// and esbuild resolving from a directory that does not exist fails
|
|
4657
|
+
// even the absolute import above.
|
|
4658
|
+
resolveDir: path9.dirname(dash.tsxPath ?? path9.join(dash.root, "dashboards", "x"))
|
|
4231
4659
|
};
|
|
4232
4660
|
});
|
|
4233
4661
|
b.onResolve({ filter: /^@malloyyo\/dashboard$/ }, () => ({ path: runtimeIndex }));
|
|
@@ -4236,23 +4664,26 @@ boot(Dashboard);
|
|
|
4236
4664
|
hostAliasPlugin
|
|
4237
4665
|
]
|
|
4238
4666
|
});
|
|
4239
|
-
|
|
4667
|
+
fs8.writeFileSync(path9.join(outDir, "assets", "site.css"), SITE_CSS);
|
|
4240
4668
|
for (const d of dashboards) {
|
|
4241
4669
|
if (rendersNoData(d)) continue;
|
|
4242
4670
|
let specs = [];
|
|
4243
4671
|
let tileSpecs;
|
|
4244
4672
|
if (d.tiles && d.entryFile) {
|
|
4245
|
-
const t = await runner.dashboardTiles(d.entryFile, d.tiles);
|
|
4673
|
+
const t = await d.runner.dashboardTiles(d.entryFile, d.tiles);
|
|
4246
4674
|
specs = t.union;
|
|
4247
4675
|
tileSpecs = t.tiles;
|
|
4248
4676
|
} else {
|
|
4249
|
-
const got = d.entryFile ? await runner.givensForQueryIn(d.entryFile, d.query) : await runner.givensForQuery(d.query);
|
|
4677
|
+
const got = d.entryFile ? await d.runner.givensForQueryIn(d.entryFile, d.query) : await d.runner.givensForQuery(d.query);
|
|
4250
4678
|
if (!got.ok) throw new Error(`dashboard ${d.name}: ${got.error}`);
|
|
4251
4679
|
specs = got.givens;
|
|
4252
4680
|
}
|
|
4253
|
-
|
|
4681
|
+
fs8.writeFileSync(
|
|
4682
|
+
path9.join(outDir, `${pageFile(d.slug)}.html`),
|
|
4683
|
+
page(d, dashboards, title, specs, tileSpecs, cleanUrls, analytics)
|
|
4684
|
+
);
|
|
4254
4685
|
}
|
|
4255
|
-
|
|
4686
|
+
fs8.writeFileSync(path9.join(outDir, "index.html"), indexPage(dashboards, title, !!landing, cleanUrls, analytics));
|
|
4256
4687
|
if (landing) {
|
|
4257
4688
|
await esbuild3.build({
|
|
4258
4689
|
stdin: {
|
|
@@ -4261,13 +4692,13 @@ import { createRoot } from "react-dom/client";
|
|
|
4261
4692
|
import Landing from ${JSON.stringify(landing)};
|
|
4262
4693
|
createRoot(document.getElementById("root")).render(React.createElement(Landing, { dashboards: window.__DASHBOARDS__ || [] }));
|
|
4263
4694
|
`,
|
|
4264
|
-
resolveDir:
|
|
4695
|
+
resolveDir: path9.dirname(landing),
|
|
4265
4696
|
loader: "js"
|
|
4266
4697
|
},
|
|
4267
4698
|
bundle: true,
|
|
4268
4699
|
format: "esm",
|
|
4269
4700
|
minify: true,
|
|
4270
|
-
outfile:
|
|
4701
|
+
outfile: path9.join(outDir, "assets", "index.js"),
|
|
4271
4702
|
logLevel: "warning",
|
|
4272
4703
|
// Same base as the dashboard pass. A landing page needs no Malloy today,
|
|
4273
4704
|
// but one that imported anything reaching antlr4ts would otherwise die at
|
|
@@ -4277,13 +4708,13 @@ createRoot(document.getElementById("root")).render(React.createElement(Landing,
|
|
|
4277
4708
|
});
|
|
4278
4709
|
}
|
|
4279
4710
|
const duck = selfHostDuckdb ? copyDuckDBAssets(outDir) : [];
|
|
4280
|
-
if (!selfHostDuckdb)
|
|
4281
|
-
const bytes = (p) =>
|
|
4282
|
-
const assetDir =
|
|
4283
|
-
const jsTotal =
|
|
4711
|
+
if (!selfHostDuckdb) fs8.rmSync(path9.join(outDir, "duckdb"), { recursive: true, force: true });
|
|
4712
|
+
const bytes = (p) => fs8.statSync(p).size;
|
|
4713
|
+
const assetDir = path9.join(outDir, "assets");
|
|
4714
|
+
const jsTotal = fs8.readdirSync(assetDir).filter((f) => f.endsWith(".js")).reduce((a, f) => a + bytes(path9.join(assetDir, f)), 0);
|
|
4284
4715
|
console.log(`
|
|
4285
|
-
bundled ${dashboards.length} dashboard(s) \u2192 ${
|
|
4286
|
-
for (const d of dashboards) console.log(` ${d.
|
|
4716
|
+
bundled ${dashboards.length} dashboard(s) \u2192 ${path9.relative(process.cwd(), outDir) || "."}/`);
|
|
4717
|
+
for (const d of dashboards) console.log(` ${pageFile(d.slug)}.html ${d.title ?? ""}`);
|
|
4287
4718
|
console.log(`
|
|
4288
4719
|
js ${(jsTotal / 1048576).toFixed(2)} MB`);
|
|
4289
4720
|
console.log(
|
|
@@ -4292,13 +4723,13 @@ bundled ${dashboards.length} dashboard(s) \u2192 ${path8.relative(process.cwd(),
|
|
|
4292
4723
|
console.log(` model ${Object.keys(modelFiles).length} .malloy files inlined`);
|
|
4293
4724
|
if (analytics) console.log(` analytics ${analytics}`);
|
|
4294
4725
|
if (copiedData.length) {
|
|
4295
|
-
const mb = copiedData.reduce((a, r) => a +
|
|
4726
|
+
const mb = copiedData.reduce((a, r) => a + fs8.statSync(path9.join(root, r)).size, 0) / 1048576;
|
|
4296
4727
|
console.log(` data ${copiedData.length} file(s) copied, ${mb.toFixed(1)} MB (same-origin)`);
|
|
4297
4728
|
}
|
|
4298
4729
|
console.log(
|
|
4299
4730
|
target === "pages" ? `
|
|
4300
|
-
Publish: commit ${
|
|
4301
|
-
Publish: deploy ${
|
|
4731
|
+
Publish: commit ${path9.basename(outDir)}/ and point GitHub Pages at it.` : `
|
|
4732
|
+
Publish: deploy ${path9.basename(outDir)}/ to Vercel (vercel.json written: clean URLs + asset caching).`
|
|
4302
4733
|
);
|
|
4303
4734
|
if (opts.serve !== false) {
|
|
4304
4735
|
await serveStatic(outDir, opts.port ?? 4180);
|
|
@@ -4308,8 +4739,9 @@ Publish: deploy ${path8.basename(outDir)}/ to Vercel (vercel.json written: clean
|
|
|
4308
4739
|
}
|
|
4309
4740
|
|
|
4310
4741
|
// src/init.ts
|
|
4311
|
-
import
|
|
4312
|
-
import
|
|
4742
|
+
import fs9 from "node:fs";
|
|
4743
|
+
import { execFileSync as execFileSync2 } from "node:child_process";
|
|
4744
|
+
import path10 from "node:path";
|
|
4313
4745
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
4314
4746
|
var AUTHOR_SERVER = "malloyyo_author";
|
|
4315
4747
|
var AUTHOR_MCP = {
|
|
@@ -4344,11 +4776,11 @@ function withAuthorPermissions(input) {
|
|
|
4344
4776
|
return { settings, added };
|
|
4345
4777
|
}
|
|
4346
4778
|
function allowAuthorTools(root) {
|
|
4347
|
-
const file =
|
|
4779
|
+
const file = path10.join(root, ".claude", "settings.json");
|
|
4348
4780
|
let existing;
|
|
4349
|
-
if (
|
|
4781
|
+
if (fs9.existsSync(file)) {
|
|
4350
4782
|
try {
|
|
4351
|
-
existing = JSON.parse(
|
|
4783
|
+
existing = JSON.parse(fs9.readFileSync(file, "utf8"));
|
|
4352
4784
|
} catch {
|
|
4353
4785
|
return { added: [], note: ".claude/settings.json isn't valid JSON \u2014 left as-is" };
|
|
4354
4786
|
}
|
|
@@ -4356,113 +4788,111 @@ function allowAuthorTools(root) {
|
|
|
4356
4788
|
const merged = withAuthorPermissions(existing);
|
|
4357
4789
|
if ("error" in merged) return { added: [], note: `${merged.error} \u2014 left as-is` };
|
|
4358
4790
|
if (merged.added.length === 0) return { added: [] };
|
|
4359
|
-
|
|
4360
|
-
|
|
4791
|
+
fs9.mkdirSync(path10.dirname(file), { recursive: true });
|
|
4792
|
+
fs9.writeFileSync(file, JSON.stringify(merged.settings, null, 2) + "\n");
|
|
4361
4793
|
return { added: merged.added };
|
|
4362
4794
|
}
|
|
4363
|
-
function
|
|
4364
|
-
const
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
for (const m of code.matchAll(/^\s*([A-Za-z_]\w*)\s*::/gm)) {
|
|
4370
|
-
names.add(m[1]);
|
|
4371
|
-
}
|
|
4372
|
-
return [...names];
|
|
4373
|
-
}
|
|
4374
|
-
function scaffoldIndex(root) {
|
|
4375
|
-
const indexPath = path9.join(root, "index.malloy");
|
|
4376
|
-
if (fs8.existsSync(indexPath)) {
|
|
4377
|
-
return { wrote: false, note: "index.malloy already exists \u2014 left as-is" };
|
|
4378
|
-
}
|
|
4379
|
-
const models = fs8.readdirSync(root).filter((f) => f.endsWith(".malloy") && f !== "index.malloy").sort();
|
|
4380
|
-
if (models.length === 0) {
|
|
4381
|
-
return { wrote: false, note: "no .malloy files found \u2014 skipped index.malloy" };
|
|
4382
|
-
}
|
|
4383
|
-
const blocks = [
|
|
4384
|
-
"// Generated by `malloyyo init` \u2014 the entry model. `dashboard dev`, `lint`,",
|
|
4385
|
-
"// `malloyyo test`, and the hosted app only see what THIS file exports.",
|
|
4386
|
-
"// Review the re-exports below (add givens / suggest queries your dashboards",
|
|
4387
|
-
"// reference), then validate with `malloyyo mcp --develop` or `dashboard dev`.",
|
|
4388
|
-
""
|
|
4389
|
-
];
|
|
4390
|
-
let anyNames = false;
|
|
4391
|
-
for (const file of models) {
|
|
4392
|
-
const names = exportableNames(fs8.readFileSync(path9.join(root, file), "utf8"));
|
|
4393
|
-
if (names.length === 0) {
|
|
4394
|
-
blocks.push(`// ${file}: no top-level source/query/given detected \u2014 add exports by hand`);
|
|
4395
|
-
continue;
|
|
4396
|
-
}
|
|
4397
|
-
anyNames = true;
|
|
4398
|
-
const list = names.join(", ");
|
|
4399
|
-
blocks.push(`import { ${list} } from './${file}'`);
|
|
4400
|
-
blocks.push(`export { ${list} }`);
|
|
4401
|
-
blocks.push("");
|
|
4795
|
+
function scaffoldRepo(root) {
|
|
4796
|
+
const notes = [];
|
|
4797
|
+
if (fs9.existsSync(path10.join(root, "index.malloy"))) {
|
|
4798
|
+
notes.push("\u2022 index.malloy at the root \u2014 the old single-dataset layout, left as-is");
|
|
4799
|
+
notes.push(" (run `malloyyo lint` here for how to convert it)");
|
|
4800
|
+
return { notes, oldLayout: true };
|
|
4402
4801
|
}
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4802
|
+
const configPath = path10.join(root, "malloy-config.json");
|
|
4803
|
+
if (fs9.existsSync(configPath)) {
|
|
4804
|
+
notes.push("\u2022 malloy-config.json exists \u2014 left as-is");
|
|
4805
|
+
} else {
|
|
4806
|
+
fs9.writeFileSync(
|
|
4807
|
+
configPath,
|
|
4808
|
+
JSON.stringify({ connections: { duckdb: { is: "duckdb" } } }, null, 2) + "\n"
|
|
4809
|
+
);
|
|
4810
|
+
notes.push("\u2713 wrote malloy-config.json \u2014 DuckDB to start; add connections as you need them");
|
|
4811
|
+
}
|
|
4812
|
+
const datasetsDir = path10.join(root, DATASETS_DIR);
|
|
4813
|
+
if (fs9.existsSync(datasetsDir)) {
|
|
4814
|
+
const have = fs9.readdirSync(datasetsDir, { withFileTypes: true }).filter((e) => e.isDirectory()).map((e) => e.name);
|
|
4815
|
+
notes.push(
|
|
4816
|
+
have.length ? `\u2022 ${DATASETS_DIR}/ exists \u2014 ${have.length} dataset(s): ${have.join(", ")}` : `\u2022 ${DATASETS_DIR}/ exists but is empty \u2014 ask claude to add a dataset`
|
|
4817
|
+
);
|
|
4818
|
+
} else {
|
|
4819
|
+
fs9.mkdirSync(datasetsDir, { recursive: true });
|
|
4820
|
+
fs9.writeFileSync(
|
|
4821
|
+
path10.join(datasetsDir, ".gitkeep"),
|
|
4822
|
+
"# Each directory here is a dataset, named after the directory.\n"
|
|
4823
|
+
);
|
|
4824
|
+
notes.push(`\u2713 created ${DATASETS_DIR}/ \u2014 one directory per dataset`);
|
|
4825
|
+
}
|
|
4826
|
+
return { notes };
|
|
4408
4827
|
}
|
|
4409
4828
|
function templatePath(...parts) {
|
|
4410
|
-
const distDir =
|
|
4829
|
+
const distDir = path10.dirname(fileURLToPath2(import.meta.url));
|
|
4411
4830
|
return [
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
].find((p) =>
|
|
4831
|
+
path10.join(distDir, "templates", ...parts),
|
|
4832
|
+
path10.join(distDir, "..", "src", "templates", ...parts)
|
|
4833
|
+
].find((p) => fs9.existsSync(p));
|
|
4415
4834
|
}
|
|
4416
4835
|
function installSkills(root) {
|
|
4417
4836
|
const srcSkills = templatePath("skills");
|
|
4418
4837
|
if (!srcSkills) return { wrote: [], skipped: [], note: "no skill templates found \u2014 skipped" };
|
|
4419
|
-
const destSkills =
|
|
4420
|
-
|
|
4838
|
+
const destSkills = path10.join(root, ".claude", "skills");
|
|
4839
|
+
fs9.mkdirSync(destSkills, { recursive: true });
|
|
4421
4840
|
const wrote = [];
|
|
4422
4841
|
const skipped = [];
|
|
4423
|
-
for (const name of
|
|
4424
|
-
const from =
|
|
4425
|
-
if (!
|
|
4426
|
-
const to =
|
|
4427
|
-
if (
|
|
4842
|
+
for (const name of fs9.readdirSync(srcSkills)) {
|
|
4843
|
+
const from = path10.join(srcSkills, name);
|
|
4844
|
+
if (!fs9.statSync(from).isDirectory()) continue;
|
|
4845
|
+
const to = path10.join(destSkills, name);
|
|
4846
|
+
if (fs9.existsSync(to)) {
|
|
4428
4847
|
skipped.push(name);
|
|
4429
4848
|
continue;
|
|
4430
4849
|
}
|
|
4431
|
-
|
|
4850
|
+
fs9.cpSync(from, to, { recursive: true });
|
|
4432
4851
|
wrote.push(name);
|
|
4433
4852
|
}
|
|
4434
4853
|
return { wrote, skipped };
|
|
4435
4854
|
}
|
|
4436
4855
|
function installDevcontainer(root) {
|
|
4437
|
-
const dest =
|
|
4438
|
-
if (
|
|
4856
|
+
const dest = path10.join(root, ".devcontainer", "devcontainer.json");
|
|
4857
|
+
if (fs9.existsSync(dest)) {
|
|
4439
4858
|
return { wrote: false, note: ".devcontainer/devcontainer.json exists \u2014 left as-is" };
|
|
4440
4859
|
}
|
|
4441
4860
|
const src = templatePath("devcontainer", "devcontainer.json");
|
|
4442
4861
|
if (!src) return { wrote: false, note: "no devcontainer template found \u2014 skipped" };
|
|
4443
|
-
|
|
4444
|
-
|
|
4862
|
+
fs9.mkdirSync(path10.dirname(dest), { recursive: true });
|
|
4863
|
+
fs9.copyFileSync(src, dest);
|
|
4445
4864
|
return {
|
|
4446
4865
|
wrote: true,
|
|
4447
4866
|
note: "wrote .devcontainer/devcontainer.json \u2014 commit it, then Code \u2192 Codespaces on GitHub"
|
|
4448
4867
|
};
|
|
4449
4868
|
}
|
|
4869
|
+
function ensureGitRepo(root) {
|
|
4870
|
+
if (fs9.existsSync(path10.join(root, ".git"))) return null;
|
|
4871
|
+
try {
|
|
4872
|
+
execFileSync2("git", ["init", "-q"], { cwd: root, stdio: ["ignore", "pipe", "pipe"] });
|
|
4873
|
+
return "\u2713 git init \u2014 a repo publishes what git tracks";
|
|
4874
|
+
} catch {
|
|
4875
|
+
return "\u2022 could not run `git init` \u2014 publishing a repo needs one, so run it yourself";
|
|
4876
|
+
}
|
|
4877
|
+
}
|
|
4450
4878
|
async function initCmd(dir) {
|
|
4451
|
-
const root =
|
|
4452
|
-
if (!
|
|
4879
|
+
const root = path10.resolve(dir);
|
|
4880
|
+
if (!fs9.existsSync(root) || !fs9.statSync(root).isDirectory()) {
|
|
4453
4881
|
throw new Error(`not a directory: ${root}`);
|
|
4454
4882
|
}
|
|
4455
|
-
const
|
|
4456
|
-
if (
|
|
4883
|
+
const gitNote = ensureGitRepo(root);
|
|
4884
|
+
if (gitNote) console.log(gitNote);
|
|
4885
|
+
const mcpPath = path10.join(root, ".mcp.json");
|
|
4886
|
+
if (fs9.existsSync(mcpPath)) {
|
|
4457
4887
|
console.log(`\u2022 .mcp.json exists \u2014 leaving it. For author-by-default it should be:`);
|
|
4458
4888
|
console.log(` ${JSON.stringify(AUTHOR_MCP.mcpServers.malloyyo_author)}`);
|
|
4459
4889
|
console.log(` (server key "malloyyo_author", command "malloyyo mcp --develop").`);
|
|
4460
4890
|
} else {
|
|
4461
|
-
|
|
4891
|
+
fs9.writeFileSync(mcpPath, JSON.stringify(AUTHOR_MCP, null, 2) + "\n");
|
|
4462
4892
|
console.log(`\u2713 wrote .mcp.json \u2014 \`cd ${dir} && claude\` now opens in AUTHOR mode`);
|
|
4463
4893
|
}
|
|
4464
|
-
const
|
|
4465
|
-
|
|
4894
|
+
const repo = scaffoldRepo(root);
|
|
4895
|
+
for (const note of repo.notes) console.log(note);
|
|
4466
4896
|
const sk = installSkills(root);
|
|
4467
4897
|
if (sk.note) {
|
|
4468
4898
|
console.log(`\u2022 ${sk.note}`);
|
|
@@ -4488,14 +4918,24 @@ async function initCmd(dir) {
|
|
|
4488
4918
|
}
|
|
4489
4919
|
console.log("");
|
|
4490
4920
|
console.log("Next:");
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4921
|
+
if (repo.oldLayout) {
|
|
4922
|
+
console.log(" claude # author mode");
|
|
4923
|
+
console.log(" malloyyo lint # what this repo publishes");
|
|
4924
|
+
console.log(" malloyyo dashboard dev # see it render");
|
|
4925
|
+
console.log(" malloyyo publish --dataset <name>");
|
|
4926
|
+
return;
|
|
4927
|
+
}
|
|
4928
|
+
console.log(" claude # author mode \u2014 ask it to add a dataset");
|
|
4929
|
+
console.log("");
|
|
4930
|
+
console.log("Once a dataset exists:");
|
|
4931
|
+
console.log(" malloyyo lint # what this repo publishes");
|
|
4932
|
+
console.log(" malloyyo dashboard dev -C datasets/<name> # see it render");
|
|
4933
|
+
console.log(" malloyyo publish --repo <owner/name> --create-datasets");
|
|
4494
4934
|
}
|
|
4495
4935
|
|
|
4496
4936
|
// src/sql.ts
|
|
4497
|
-
import
|
|
4498
|
-
import
|
|
4937
|
+
import fs10 from "node:fs";
|
|
4938
|
+
import path11 from "node:path";
|
|
4499
4939
|
import url5 from "node:url";
|
|
4500
4940
|
import { MalloyConfig as MalloyConfig3, Runtime as Runtime3, discoverConfig as discoverConfig3 } from "@malloydata/malloy";
|
|
4501
4941
|
function fileReader() {
|
|
@@ -4504,12 +4944,12 @@ function fileReader() {
|
|
|
4504
4944
|
if (u.protocol !== "file:") {
|
|
4505
4945
|
throw new Error(`unsupported URL scheme for import: ${u.href}`);
|
|
4506
4946
|
}
|
|
4507
|
-
return
|
|
4947
|
+
return fs10.promises.readFile(u, "utf8");
|
|
4508
4948
|
}
|
|
4509
4949
|
};
|
|
4510
4950
|
}
|
|
4511
4951
|
async function loadConfig3(rootDir) {
|
|
4512
|
-
const rootUrl = url5.pathToFileURL(rootDir.endsWith(
|
|
4952
|
+
const rootUrl = url5.pathToFileURL(rootDir.endsWith(path11.sep) ? rootDir : rootDir + path11.sep);
|
|
4513
4953
|
const discovered = await discoverConfig3(rootUrl, rootUrl, fileReader()).catch(() => null);
|
|
4514
4954
|
return discovered ?? new MalloyConfig3({ includeDefaultConnections: true }, {
|
|
4515
4955
|
rootDirectory: rootUrl.toString()
|
|
@@ -4540,12 +4980,12 @@ async function readStdin() {
|
|
|
4540
4980
|
}
|
|
4541
4981
|
async function resolveSql(opts) {
|
|
4542
4982
|
if (opts.execute != null) return opts.execute;
|
|
4543
|
-
if (opts.file) return
|
|
4983
|
+
if (opts.file) return fs10.promises.readFile(opts.file, "utf8");
|
|
4544
4984
|
return readStdin();
|
|
4545
4985
|
}
|
|
4546
4986
|
async function sqlCmd(connection, opts) {
|
|
4547
4987
|
const name = connection ?? "duckdb";
|
|
4548
|
-
const rootDir =
|
|
4988
|
+
const rootDir = path11.resolve(opts.root ?? ".");
|
|
4549
4989
|
const sql = (await resolveSql(opts)).trim();
|
|
4550
4990
|
if (!sql) {
|
|
4551
4991
|
throw new Error("no SQL provided \u2014 pass -e <sql>, -f <file>, or pipe it via stdin");
|
|
@@ -4573,22 +5013,22 @@ async function sqlCmd(connection, opts) {
|
|
|
4573
5013
|
|
|
4574
5014
|
// src/launch.ts
|
|
4575
5015
|
import { spawn as spawn2 } from "node:child_process";
|
|
4576
|
-
import
|
|
5016
|
+
import fs11 from "node:fs";
|
|
4577
5017
|
import os from "node:os";
|
|
4578
|
-
import
|
|
5018
|
+
import path12 from "node:path";
|
|
4579
5019
|
var SURFACE_FLAG = { author: "--develop", test: "--explore" };
|
|
4580
5020
|
var SERVER_KEY = { author: "malloyyo_author", test: "malloyyo_test" };
|
|
4581
5021
|
async function launchCmd(mode, opts) {
|
|
4582
|
-
const root =
|
|
4583
|
-
const tmpDir =
|
|
4584
|
-
const cfgPath =
|
|
5022
|
+
const root = path12.resolve(opts.root ?? process.cwd());
|
|
5023
|
+
const tmpDir = fs11.mkdtempSync(path12.join(os.tmpdir(), "malloyyo-launch-"));
|
|
5024
|
+
const cfgPath = path12.join(tmpDir, "mcp.json");
|
|
4585
5025
|
const cfg = {
|
|
4586
5026
|
mcpServers: {
|
|
4587
5027
|
// Absolute -C: an ephemeral config, so pinning the root is robust.
|
|
4588
5028
|
[SERVER_KEY[mode]]: { command: "malloyyo", args: ["mcp", SURFACE_FLAG[mode], "-C", root] }
|
|
4589
5029
|
}
|
|
4590
5030
|
};
|
|
4591
|
-
|
|
5031
|
+
fs11.writeFileSync(cfgPath, JSON.stringify(cfg, null, 2));
|
|
4592
5032
|
const label = mode === "author" ? "AUTHOR (compile/edit)" : "TEST (claude.ai web preview)";
|
|
4593
5033
|
process.stderr.write(`\u25B6 launching Claude in ${label} mode over ${root}
|
|
4594
5034
|
`);
|
|
@@ -4607,7 +5047,7 @@ async function launchCmd(mode, opts) {
|
|
|
4607
5047
|
});
|
|
4608
5048
|
child.on("exit", () => resolve5());
|
|
4609
5049
|
});
|
|
4610
|
-
|
|
5050
|
+
fs11.rmSync(tmpDir, { recursive: true, force: true });
|
|
4611
5051
|
}
|
|
4612
5052
|
|
|
4613
5053
|
// src/draft.ts
|
|
@@ -4644,8 +5084,8 @@ var DraftRequestError = class extends Error {
|
|
|
4644
5084
|
this.status = status2;
|
|
4645
5085
|
}
|
|
4646
5086
|
};
|
|
4647
|
-
async function draftFetch(t,
|
|
4648
|
-
const res = await apiFetch(`${t.url}/api/datasets/${encodeURIComponent(t.dataset)}${
|
|
5087
|
+
async function draftFetch(t, path13, bearer, init) {
|
|
5088
|
+
const res = await apiFetch(`${t.url}/api/datasets/${encodeURIComponent(t.dataset)}${path13}`, {
|
|
4649
5089
|
...init,
|
|
4650
5090
|
headers: { ...init?.headers ?? {}, authorization: `Bearer ${bearer}` }
|
|
4651
5091
|
});
|
|
@@ -4812,7 +5252,7 @@ function createControlPlaneClient(options) {
|
|
|
4812
5252
|
const base = options.apiUrl.replace(/\/+$/, "");
|
|
4813
5253
|
const sleep = options.sleep ?? ((ms) => new Promise((resolve5) => setTimeout(resolve5, ms)));
|
|
4814
5254
|
const maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
4815
|
-
async function attempt(method,
|
|
5255
|
+
async function attempt(method, path13, extra) {
|
|
4816
5256
|
const token = await options.getAccessToken();
|
|
4817
5257
|
const headers = {
|
|
4818
5258
|
authorization: `Bearer ${token}`,
|
|
@@ -4820,7 +5260,7 @@ function createControlPlaneClient(options) {
|
|
|
4820
5260
|
};
|
|
4821
5261
|
if (extra?.idempotencyKey !== void 0) headers["idempotency-key"] = extra.idempotencyKey;
|
|
4822
5262
|
if (extra?.body !== void 0) headers["content-type"] = "application/json";
|
|
4823
|
-
const response = await doFetch(`${base}${
|
|
5263
|
+
const response = await doFetch(`${base}${path13}`, {
|
|
4824
5264
|
method,
|
|
4825
5265
|
headers,
|
|
4826
5266
|
...extra?.body === void 0 ? {} : { body: JSON.stringify(extra.body) }
|
|
@@ -4837,10 +5277,10 @@ function createControlPlaneClient(options) {
|
|
|
4837
5277
|
}
|
|
4838
5278
|
return parsed;
|
|
4839
5279
|
}
|
|
4840
|
-
async function request(method,
|
|
5280
|
+
async function request(method, path13, extra) {
|
|
4841
5281
|
for (let attemptNumber = 1; ; attemptNumber += 1) {
|
|
4842
5282
|
try {
|
|
4843
|
-
return await attempt(method,
|
|
5283
|
+
return await attempt(method, path13, extra);
|
|
4844
5284
|
} catch (error) {
|
|
4845
5285
|
const transient = error instanceof ControlPlaneError ? isTransientStatus(error.status) : isNetworkError(error);
|
|
4846
5286
|
if (!transient || attemptNumber >= maxAttempts) throw error;
|
|
@@ -5672,7 +6112,23 @@ function requestFailed(what, res, out, t, source, bearer) {
|
|
|
5672
6112
|
const hint = res.status === 401 || res.status === 403 ? authHint(res.status, t, source, foreign) : failureHint(out, t);
|
|
5673
6113
|
return new Error(`${what} failed: ${detail}${hint}`);
|
|
5674
6114
|
}
|
|
5675
|
-
var PUBLISH_HELP = `
|
|
6115
|
+
var PUBLISH_HELP = `One dataset, or several:
|
|
6116
|
+
The repo's shape decides, so you don't declare it:
|
|
6117
|
+
|
|
6118
|
+
index.malloy one dataset
|
|
6119
|
+
dashboards/ publish --dataset movies
|
|
6120
|
+
--create-dataset to make it
|
|
6121
|
+
|
|
6122
|
+
datasets/finance/index.malloy one dataset per directory
|
|
6123
|
+
datasets/sales/index.malloy publish --repo owner/name
|
|
6124
|
+
malloy-config.json --create-datasets to make them
|
|
6125
|
+
|
|
6126
|
+
Use the wrong flag for your layout and the error says which to use.
|
|
6127
|
+
|
|
6128
|
+
A repo publishes as a unit: every dataset is linted, and the server writes
|
|
6129
|
+
all of them or none. \`malloyyo lint\` shows what yours publishes.
|
|
6130
|
+
|
|
6131
|
+
Target resolution:
|
|
5676
6132
|
The instance and dataset normally come from the \`malloyyo\` block in
|
|
5677
6133
|
malloy-config.json. Either can be overridden, and giving BOTH means the
|
|
5678
6134
|
config is never read \u2014 so a repo with no targets (or none for this
|
|
@@ -5684,12 +6140,99 @@ var PUBLISH_HELP = `Target resolution:
|
|
|
5684
6140
|
whose url should be borrowed. Authenticate the same way as always:
|
|
5685
6141
|
\`malloyyo login <url>\`, or set $MALLOYYO_TOKEN (mint one at
|
|
5686
6142
|
<url>/settings/tokens \u2014 that is what CI wants), or pass --token.`;
|
|
6143
|
+
async function publishRepo(root, datasetNames, target, opts) {
|
|
6144
|
+
const t = resolvePublishTarget(root, target, { instance: opts.instance, dataset: opts.repo });
|
|
6145
|
+
const bearer = await getAccessToken(t, { tokenFlag: opts.token });
|
|
6146
|
+
if (!opts.skipLint) {
|
|
6147
|
+
const linted = await lintRepo(root);
|
|
6148
|
+
console.log("dashboards:");
|
|
6149
|
+
printRepoLintReport(linted);
|
|
6150
|
+
if (!linted.ok) {
|
|
6151
|
+
throw new Error(
|
|
6152
|
+
"dashboard lint failed \u2014 fix the above, or pass --skip-lint" + missingEnvHint(missingEnvRefs(gatherDirectory(root).config), "this shell")
|
|
6153
|
+
);
|
|
6154
|
+
}
|
|
6155
|
+
}
|
|
6156
|
+
const { zip: archive, fileCount } = gitArchiveZip(root);
|
|
6157
|
+
if (fileCount === 0) throw new Error(`${root} has no files git tracks \u2014 nothing to publish`);
|
|
6158
|
+
const git = gitInfo(root);
|
|
6159
|
+
const provenance = git.sha ? `${git.branch}@${shortSha(git.sha)}${git.dirty ? " (dirty)" : ""}` : "(no git)";
|
|
6160
|
+
console.log(`\u2192 ${t.url} repo=${opts.repo}${opts.createDatasets ? " (create missing)" : ""}`);
|
|
6161
|
+
console.log(` ${datasetNames.length} dataset(s): ${datasetNames.join(", ")}`);
|
|
6162
|
+
console.log(` ${fileCount} file(s), ${(archive.length / 1024).toFixed(0)}KB archive ${provenance}`);
|
|
6163
|
+
if (opts.dryRun) {
|
|
6164
|
+
console.log("dry run \u2014 not sending");
|
|
6165
|
+
return;
|
|
6166
|
+
}
|
|
6167
|
+
const res = await apiFetch(`${t.url}/api/repos/push`, {
|
|
6168
|
+
method: "POST",
|
|
6169
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${bearer}` },
|
|
6170
|
+
body: JSON.stringify({
|
|
6171
|
+
repo: opts.repo,
|
|
6172
|
+
branch: git.branch ?? "main",
|
|
6173
|
+
archive: archive.toString("base64"),
|
|
6174
|
+
createDatasets: opts.createDatasets ?? false,
|
|
6175
|
+
git
|
|
6176
|
+
})
|
|
6177
|
+
});
|
|
6178
|
+
const out = await res.json().catch(() => null);
|
|
6179
|
+
if (!res.ok || !out?.ok) {
|
|
6180
|
+
throw new Error(out?.error ?? `publish failed (${res.status})`);
|
|
6181
|
+
}
|
|
6182
|
+
for (const d of out.datasets ?? []) {
|
|
6183
|
+
console.log(` \u2713 ${d.name} v${d.version}${d.created ? " (created)" : ""}`);
|
|
6184
|
+
}
|
|
6185
|
+
console.log(`published ${out.datasets?.length ?? 0} dataset(s) to ${t.url}`);
|
|
6186
|
+
}
|
|
5687
6187
|
async function publish(target, dir, opts) {
|
|
5688
6188
|
const root = resolve4(dir);
|
|
6189
|
+
if (target && /^https?:\/\//i.test(target)) {
|
|
6190
|
+
opts = { ...opts, instance: opts.instance ?? target };
|
|
6191
|
+
target = void 0;
|
|
6192
|
+
}
|
|
6193
|
+
const layout = await layoutFromListing(fsLister(root), root);
|
|
6194
|
+
if (!layout.ok) throw new Error(layout.error);
|
|
6195
|
+
const oldLayout = layout.kind === "single" && repoRootOf(root) === root;
|
|
6196
|
+
if (layout.kind !== "single") {
|
|
6197
|
+
const names = layout.datasets.map((d) => d.name);
|
|
6198
|
+
if (opts.dataset) {
|
|
6199
|
+
throw new Error(
|
|
6200
|
+
`${root} publishes ${names.length} datasets (${names.join(", ")}), so --dataset cannot say which.
|
|
6201
|
+
Publish the repo instead: malloyyo publish --repo <owner/name>` + (opts.createDataset ? " --create-datasets" : "")
|
|
6202
|
+
);
|
|
6203
|
+
}
|
|
6204
|
+
if (!opts.repo) {
|
|
6205
|
+
throw new Error(
|
|
6206
|
+
`${root} publishes ${names.length} datasets, and a repo is published as one unit.
|
|
6207
|
+
Name it: malloyyo publish --repo <owner/name>`
|
|
6208
|
+
);
|
|
6209
|
+
}
|
|
6210
|
+
return publishRepo(root, names, target, opts);
|
|
6211
|
+
}
|
|
6212
|
+
const repoRoot = repoRootOf(root);
|
|
6213
|
+
if (repoRoot !== root) {
|
|
6214
|
+
const rel = relative2(repoRoot, root).split(sep2).join("/");
|
|
6215
|
+
throw new Error(
|
|
6216
|
+
`${rel} is one dataset of the repo above it, and a repo publishes as one unit.
|
|
6217
|
+
Publish the repo: cd ${repoRoot} && malloyyo publish --repo <owner/name>`
|
|
6218
|
+
);
|
|
6219
|
+
}
|
|
6220
|
+
if (opts.repo) {
|
|
6221
|
+
throw new Error(
|
|
6222
|
+
`${root} publishes a single dataset (index.malloy at its root), so --repo has nothing to name.
|
|
6223
|
+
Use --dataset <name>.`
|
|
6224
|
+
);
|
|
6225
|
+
}
|
|
6226
|
+
if (opts.createDatasets) {
|
|
6227
|
+
throw new Error("--create-datasets is for a repo with a datasets/ directory; use --create-dataset.");
|
|
6228
|
+
}
|
|
5689
6229
|
const t = resolvePublishTarget(root, target, {
|
|
5690
6230
|
instance: opts.instance,
|
|
5691
6231
|
dataset: opts.dataset
|
|
5692
6232
|
});
|
|
6233
|
+
if (oldLayout) console.log(`
|
|
6234
|
+
${OLD_LAYOUT_NOTICE}
|
|
6235
|
+
`);
|
|
5693
6236
|
const source = tokenSource(t, { tokenFlag: opts.token });
|
|
5694
6237
|
const bearer = await getAccessToken(t, { tokenFlag: opts.token });
|
|
5695
6238
|
const { files, config } = gatherDirectory(root);
|
|
@@ -5697,7 +6240,7 @@ async function publish(target, dir, opts) {
|
|
|
5697
6240
|
throw new Error(`No .malloy files found under ${root}`);
|
|
5698
6241
|
}
|
|
5699
6242
|
if (!opts.skipLint) {
|
|
5700
|
-
const report = await lintDashboards(root);
|
|
6243
|
+
const report = await lintDashboards(root, { repoRoot: repoRootOf(root) });
|
|
5701
6244
|
if (report.dashboards.length > 0) {
|
|
5702
6245
|
console.log("dashboards:");
|
|
5703
6246
|
printLintReport(report);
|
|
@@ -5734,9 +6277,10 @@ async function publish(target, dir, opts) {
|
|
|
5734
6277
|
console.log(
|
|
5735
6278
|
`\u2713 published version ${out.version} \u2014 ${out.sources?.length ?? 0} source(s)` + (dashboards.length ? `, ${dashboards.length} dashboard(s)` : "") + (out.requiredGivens?.length ? `, scoped by ${out.requiredGivens.join(", ")}` : "")
|
|
5736
6279
|
);
|
|
5737
|
-
if (out.
|
|
6280
|
+
if (out.declaredButUnused?.length) {
|
|
6281
|
+
const many = out.declaredButUnused.length > 1;
|
|
5738
6282
|
console.log(
|
|
5739
|
-
` \u21B3 this
|
|
6283
|
+
` \u21B3 this model declares ${out.declaredButUnused.join(", ")}, which this dataset is NOT scoped by. Nothing supplies ${many ? "them" : "it"}, so ${many ? "those filters" : "that filter"} will use the declaration default. An admin ticks ${many ? "them" : "it"} on the dataset.`
|
|
5740
6284
|
);
|
|
5741
6285
|
}
|
|
5742
6286
|
}
|
|
@@ -5780,18 +6324,36 @@ program.command("publish").argument(
|
|
|
5780
6324
|
).argument("[dir]", "directory to publish", ".").option(
|
|
5781
6325
|
"-i, --instance <instance>",
|
|
5782
6326
|
"instance URL, or a configured target name to borrow the url from; overrides the config"
|
|
5783
|
-
).option("--dataset <dataset>", "dataset to publish into; overrides the config").option("--token <token>", "bearer token (overrides login/env)").option("--dry-run", "gather and report what would be sent, but don't POST").option("--skip-lint", "skip the pre-publish dashboard lint").option("--create-dataset", "create the target dataset if it doesn't exist yet (private)").
|
|
6327
|
+
).option("--dataset <dataset>", "dataset to publish into; overrides the config").option("--token <token>", "bearer token (overrides login/env)").option("--dry-run", "gather and report what would be sent, but don't POST").option("--skip-lint", "skip the pre-publish dashboard lint").option("--create-dataset", "create the target dataset if it doesn't exist yet (private)").option(
|
|
6328
|
+
"--repo <owner/name>",
|
|
6329
|
+
"publish a repo with a datasets/ directory \u2014 every dataset in it, as one unit"
|
|
6330
|
+
).option("--create-datasets", "with --repo: create any dataset the repo publishes that doesn't exist yet").description('push the Malloy model in <dir> (default ".") to <target>').addHelpText("after", `
|
|
5784
6331
|
${PUBLISH_HELP}
|
|
5785
6332
|
`).action(publish);
|
|
5786
|
-
program.command("lint").argument("[dir]", "directory to lint", ".").description("validate
|
|
6333
|
+
program.command("lint").argument("[dir]", "repo (or dataset directory) to lint", ".").description("validate the whole repo: its layout, and every dataset's dashboards").action(async (dir) => {
|
|
5787
6334
|
const root = resolve4(dir);
|
|
5788
|
-
const
|
|
5789
|
-
if (
|
|
5790
|
-
console.log("no
|
|
6335
|
+
const repo = await lintRepo(root);
|
|
6336
|
+
if (repo.empty) {
|
|
6337
|
+
console.log("no datasets yet \u2014 run `claude` here and ask it to add one.");
|
|
6338
|
+
return;
|
|
6339
|
+
}
|
|
6340
|
+
if (repo.layoutError) {
|
|
6341
|
+
console.error(`\u2717 ${repo.layoutError}`);
|
|
6342
|
+
process.exit(1);
|
|
6343
|
+
}
|
|
6344
|
+
const total = repo.datasets.reduce((n, d) => n + d.report.dashboards.length, 0);
|
|
6345
|
+
if (total === 0) {
|
|
6346
|
+
console.log(
|
|
6347
|
+
repo.datasets.length > 1 ? `${repo.datasets.length} datasets, no dashboards to lint` : "no dashboards to lint"
|
|
6348
|
+
);
|
|
6349
|
+
if (repo.oldLayout) console.log(`
|
|
6350
|
+
${OLD_LAYOUT_NOTICE}`);
|
|
5791
6351
|
return;
|
|
5792
6352
|
}
|
|
5793
|
-
|
|
5794
|
-
if (
|
|
6353
|
+
printRepoLintReport(repo);
|
|
6354
|
+
if (repo.oldLayout) console.log(`
|
|
6355
|
+
${OLD_LAYOUT_NOTICE}`);
|
|
6356
|
+
if (!repo.ok) {
|
|
5795
6357
|
const hint = missingEnvHint(missingEnvRefs(gatherDirectory(root).config), "this shell");
|
|
5796
6358
|
if (hint) console.error(hint.replace(/^\n/, ""));
|
|
5797
6359
|
process.exit(1);
|
|
@@ -5814,7 +6376,7 @@ program.command("mcp").option("-C, --root <dir>", "project root (default: curren
|
|
|
5814
6376
|
});
|
|
5815
6377
|
});
|
|
5816
6378
|
program.command("init").argument("[dir]", "model repo to set up", ".").description(
|
|
5817
|
-
"set up
|
|
6379
|
+
"set up an empty model repo: .mcp.json so `cd <repo> && claude` opens in author mode, plus malloy-config.json and datasets/. Ask claude to add datasets."
|
|
5818
6380
|
).action(initCmd);
|
|
5819
6381
|
program.command("sql").argument("[connection]", "connection name from malloy-config.json", "duckdb").option("-e, --execute <sql>", "SQL to run (else read from -f <file> or stdin)").option("-f, --file <path>", "read SQL from a file").option("-C, --root <dir>", "project root for malloy-config.json discovery (default: current directory)").option("-j, --json", "print result rows as JSON").description(
|
|
5820
6382
|
"run raw SQL against a configured connection using the embedded DuckDB \u2014 e.g. COPY a web CSV into docs/*.parquet, no standalone duckdb needed"
|