htmlineage 0.1.0-alpha.0 → 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -17
- package/package.json +3 -2
- package/skills/htmlineage/SKILL.md +173 -0
- package/skills/htmlineage/references/example/.lineage/august-retention.json +167 -0
- package/skills/htmlineage/references/example/README.md +51 -0
- package/skills/htmlineage/references/example/august-retention.html +111 -0
- package/skills/htmlineage/references/format.md +218 -0
- package/skills/htmlineage/references/review.md +79 -0
- package/src/check/rules.js +61 -19
- package/src/check/summary.js +1 -1
- package/src/commands/check.js +4 -1
- package/src/commands/view.js +6 -4
- package/src/graph/expr.js +4 -2
- package/src/graph/schema.js +41 -15
- package/src/viewer/client/bridge.js +4 -2
- package/src/viewer/client/favicon.svg +6 -0
- package/src/viewer/client/index.html +6 -6
- package/src/viewer/client/lineage.js +25 -1
- package/src/viewer/client/viewer.css +13 -11
- package/src/viewer/client/viewer.js +11 -7
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# The lineage graph: field reference
|
|
2
|
+
|
|
3
|
+
## Files
|
|
4
|
+
|
|
5
|
+
A report is one HTML file. Its lineage graph is one JSON object kept in
|
|
6
|
+
`.lineage/<report-stem>.json`, beside the report. For `report.html` that is
|
|
7
|
+
`.lineage/report.json`. You write the graph there. `htmlineage check` embeds
|
|
8
|
+
it in the report as a `<script type="application/lineage+json">` block, so you
|
|
9
|
+
never write that block by hand.
|
|
10
|
+
|
|
11
|
+
The graph is a header and a `nodes` map keyed by id:
|
|
12
|
+
|
|
13
|
+
```json
|
|
14
|
+
{
|
|
15
|
+
"lineage": "1.0",
|
|
16
|
+
"nodes": {
|
|
17
|
+
"q1": {},
|
|
18
|
+
"v1": {},
|
|
19
|
+
"f1": {},
|
|
20
|
+
"i1": {}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Kinds of node
|
|
26
|
+
|
|
27
|
+
Every value on the page reaches it by one path: source, then view, then
|
|
28
|
+
figure, then insight. A node's `step` says which it is.
|
|
29
|
+
|
|
30
|
+
```json
|
|
31
|
+
{
|
|
32
|
+
"step": "source",
|
|
33
|
+
"kind": "sql",
|
|
34
|
+
"desc": "Retention by channel and cohort month, January to August 2026",
|
|
35
|
+
"mart": "marts.retention_by_channel",
|
|
36
|
+
"period": { "from": "2026-01-01", "to": "2026-08-31" },
|
|
37
|
+
"sql": "select\n channel,\n cohort_month,\n retention_30d\nfrom marts.retention_by_channel\nwhere cohort_month between '2026-01-01' and '2026-08-31'",
|
|
38
|
+
"columns": ["channel", "cohort_month", "retention_30d"]
|
|
39
|
+
}
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
```json
|
|
43
|
+
{
|
|
44
|
+
"step": "source",
|
|
45
|
+
"kind": "external",
|
|
46
|
+
"type": "csv",
|
|
47
|
+
"ref": "q3-targets.csv",
|
|
48
|
+
"given": "A CSV with columns channel, target_retention, shared by the builder"
|
|
49
|
+
}
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
```json
|
|
53
|
+
{
|
|
54
|
+
"step": "view",
|
|
55
|
+
"from": ["q1"],
|
|
56
|
+
"desc": "Average 30-day retention for each channel and cohort month",
|
|
57
|
+
"sql": "select\n channel,\n cohort_month,\n avg(retention_30d) as retention\nfrom q1\ngroup by channel, cohort_month",
|
|
58
|
+
"columns": ["channel", "cohort_month", "retention"],
|
|
59
|
+
"encoding": { "x": "cohort_month", "y": "retention", "series": "channel" }
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
```json
|
|
64
|
+
{
|
|
65
|
+
"step": "figure",
|
|
66
|
+
"from": ["v1"],
|
|
67
|
+
"desc": "Paid social retention for August 2026",
|
|
68
|
+
"sql": "select retention\nfrom v1\nwhere channel = 'paid_social'\n and cohort_month = '2026-08-01'",
|
|
69
|
+
"value": 0.466,
|
|
70
|
+
"display": "46.6%",
|
|
71
|
+
"label": "Paid social retention, August"
|
|
72
|
+
}
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
```json
|
|
76
|
+
{
|
|
77
|
+
"step": "figure",
|
|
78
|
+
"from": ["f1", "f2"],
|
|
79
|
+
"desc": "Change in paid social retention from July to August, relative to July",
|
|
80
|
+
"expr": "(f1 - f2) / f2",
|
|
81
|
+
"value": 0.1175,
|
|
82
|
+
"display": "11.8%",
|
|
83
|
+
"label": "Change in paid social retention, August vs July"
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
```json
|
|
88
|
+
{
|
|
89
|
+
"step": "insight",
|
|
90
|
+
"from": ["f3"],
|
|
91
|
+
"text": "Paid social retention rose by a relative 11.8%."
|
|
92
|
+
}
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
A claim that rests on nothing you queried is still written down, marked
|
|
96
|
+
`"ungrounded": true` with `"from": []`.
|
|
97
|
+
|
|
98
|
+
## What may rest on what
|
|
99
|
+
|
|
100
|
+
- A view's `from` lists sources and views.
|
|
101
|
+
- A figure with `sql` lists sources and views in `from`. A figure with `expr`
|
|
102
|
+
lists figures.
|
|
103
|
+
- An insight's `from` lists views, figures and insights. Never a source:
|
|
104
|
+
sources do not appear on the page.
|
|
105
|
+
- A figure never gets its own query. It picks one cell from a view or source
|
|
106
|
+
that exists, or it derives from other figures.
|
|
107
|
+
|
|
108
|
+
## Ids
|
|
109
|
+
|
|
110
|
+
- Lowercase letters, digits and `_`, starting with a letter.
|
|
111
|
+
- Start with the step's letter: `q` for a sql source, `x` for an external
|
|
112
|
+
source, `v` for a view, `f` for a figure, `i` for an insight. For example
|
|
113
|
+
`q1`, `x1`, `v1`, `f1`, `i1`.
|
|
114
|
+
- Each id is used once.
|
|
115
|
+
|
|
116
|
+
## Binding to the page
|
|
117
|
+
|
|
118
|
+
Put `data-lineage="<id>"` on the element that shows the node:
|
|
119
|
+
|
|
120
|
+
```html
|
|
121
|
+
<span data-lineage="f3">11.8%</span>
|
|
122
|
+
<p data-lineage="i1">
|
|
123
|
+
Paid social retention rose by a relative
|
|
124
|
+
<span data-lineage="f3">11.8%</span>.
|
|
125
|
+
</p>
|
|
126
|
+
<div data-lineage="v1"><svg></svg></div>
|
|
127
|
+
<table data-lineage="v2"></table>
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
- Views, figures and insights bind. Sources never do.
|
|
131
|
+
- Every view, figure and insight binds at least once. A figure shown in
|
|
132
|
+
several places binds in each.
|
|
133
|
+
- A number inside a sentence sits in its own `data-lineage` span.
|
|
134
|
+
- A chart binds on the container around the SVG, not the SVG.
|
|
135
|
+
- The attribute adds no style and no behaviour.
|
|
136
|
+
|
|
137
|
+
## Fields
|
|
138
|
+
|
|
139
|
+
Each table below lists every field a node of that kind may carry. Required
|
|
140
|
+
means an error if it is missing. Expected means a warning if it is missing.
|
|
141
|
+
|
|
142
|
+
### sql source
|
|
143
|
+
|
|
144
|
+
| Field | Required | Type | Use |
|
|
145
|
+
| ----------- | -------- | ------ | ------------------------------------------------------------------- |
|
|
146
|
+
| `step` | required | string | `"source"` |
|
|
147
|
+
| `kind` | required | string | `"sql"` |
|
|
148
|
+
| `desc` | expected | string | One line, 120 characters at most: what the query pulls, in words |
|
|
149
|
+
| `how` | optional | string | Leave out |
|
|
150
|
+
| `mart` | expected | string | The mart the query reads |
|
|
151
|
+
| `period` | expected | object | The pinned range, `{ "from": "2026-01-01", "to": "2026-08-31" }` |
|
|
152
|
+
| `sql` | expected | string | The query, with the period as literal dates |
|
|
153
|
+
| `columns` | expected | list | Output column names |
|
|
154
|
+
| `rows` | optional | object | Up to 10 rows, columnar: `{ "column": [values] }`; or `null` |
|
|
155
|
+
| `transform` | optional | string | A judgement the SQL does not show, such as "excluded the test week" |
|
|
156
|
+
|
|
157
|
+
### external source
|
|
158
|
+
|
|
159
|
+
| Field | Required | Type | Use |
|
|
160
|
+
| ----------- | -------- | ------ | --------------------------------------------- |
|
|
161
|
+
| `step` | required | string | `"source"` |
|
|
162
|
+
| `kind` | required | string | `"external"` |
|
|
163
|
+
| `type` | expected | string | What kind of thing it is, such as `csv` |
|
|
164
|
+
| `ref` | expected | string | How a reader can find it, such as a file name |
|
|
165
|
+
| `given` | expected | string | What you were handed, in words |
|
|
166
|
+
| `columns` | optional | list | Column names, if it has them |
|
|
167
|
+
| `rows` | optional | object | Up to 10 rows, columnar; or `null` |
|
|
168
|
+
| `transform` | optional | string | A judgement the data does not show |
|
|
169
|
+
|
|
170
|
+
### view
|
|
171
|
+
|
|
172
|
+
| Field | Required | Type | Use |
|
|
173
|
+
| ------------ | -------- | ------ | -------------------------------------------------------------------------------- |
|
|
174
|
+
| `step` | required | string | `"view"` |
|
|
175
|
+
| `from` | required | list | Ids of the sources and views it reads |
|
|
176
|
+
| `desc` | expected | string | One line, 120 characters at most: what it computes and how, in words |
|
|
177
|
+
| `sql` | required | string | A SELECT over the parents, which it names by id as table names |
|
|
178
|
+
| `columns` | expected | list | Output column names |
|
|
179
|
+
| `encoding` | optional | object | For charts: `x`, `y`, `series`, `visual`, each a string |
|
|
180
|
+
| `rows` | optional | object | Up to 10 rows, columnar; or `null`. A view shown as a table needs none |
|
|
181
|
+
| `transform` | optional | string | A judgement the SQL does not show |
|
|
182
|
+
| `ungrounded` | optional | bool | `true` when it has no source: `from` may then be empty and `sql` may be left out |
|
|
183
|
+
|
|
184
|
+
### figure
|
|
185
|
+
|
|
186
|
+
| Field | Required | Type | Use |
|
|
187
|
+
| ------------ | -------------------- | ------------- | --------------------------------------------------------------------------------------------- |
|
|
188
|
+
| `step` | required | string | `"figure"` |
|
|
189
|
+
| `from` | required | list | Ids it reads: sources and views with `sql`, figures with `expr` |
|
|
190
|
+
| `desc` | expected | string | One line, 120 characters at most: how it was computed, in words |
|
|
191
|
+
| `sql` | one of `sql`, `expr` | string | A single SELECT returning one cell, ideally a plain pick of one column from one row |
|
|
192
|
+
| `expr` | one of `sql`, `expr` | string | A SQL scalar expression over figure ids, such as `(f1 - f2) / f2`. No comments |
|
|
193
|
+
| `value` | expected | number/string | The number or name the page shows |
|
|
194
|
+
| `display` | expected | string | The text exactly as shown on the page, such as `"46.6%"` |
|
|
195
|
+
| `label` | expected | string | What the number is, in words |
|
|
196
|
+
| `transform` | optional | string | A judgement the SQL does not show |
|
|
197
|
+
| `ungrounded` | optional | bool | `true` when it has no source: `from` may then be empty and neither `sql` nor `expr` is needed |
|
|
198
|
+
|
|
199
|
+
### insight
|
|
200
|
+
|
|
201
|
+
| Field | Required | Type | Use |
|
|
202
|
+
| ------------ | -------- | ------ | --------------------------------------------------------------------- |
|
|
203
|
+
| `step` | required | string | `"insight"` |
|
|
204
|
+
| `from` | required | list | Ids of the views, figures and insights the claim rests on |
|
|
205
|
+
| `text` | required | string | The claim, as it reads on the page |
|
|
206
|
+
| `transform` | optional | string | A judgement behind the claim that its sources do not show |
|
|
207
|
+
| `ungrounded` | optional | bool | `true` when the claim rests on nothing here: `from` may then be empty |
|
|
208
|
+
|
|
209
|
+
### header
|
|
210
|
+
|
|
211
|
+
| Field | Required | Type | Use |
|
|
212
|
+
| -------------- | -------- | ------ | ------------------------------------------------------ |
|
|
213
|
+
| `lineage` | required | string | The format version, `"1.0"` |
|
|
214
|
+
| `nodes` | required | object | Every node, keyed by id |
|
|
215
|
+
| `report_id` | optional | string | A UUID, minted at the first build and kept on rebuilds |
|
|
216
|
+
| `built_at` | optional | string | When the report was built, an ISO timestamp |
|
|
217
|
+
| `builder_tool` | optional | string | The tool that built it, such as `report-builder` |
|
|
218
|
+
| `signature` | optional | string | Leave out |
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# Reviewing a report
|
|
2
|
+
|
|
3
|
+
Use this when you are asked to review a report someone else built, or to check
|
|
4
|
+
one before it is shared. Your job is to show the reviewer what each number and
|
|
5
|
+
claim rests on and where that is weak. You do not decide whether the report is
|
|
6
|
+
right, and you do not change it.
|
|
7
|
+
|
|
8
|
+
## 1. Read the check summary
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
npx htmlineage check <report>.html --no-embed
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
`--no-embed` leaves the file untouched. Read the summary. It lists errors
|
|
15
|
+
last, but take them first:
|
|
16
|
+
|
|
17
|
+
- **Errors.** The lineage is broken. Say so first: nothing below can be traced
|
|
18
|
+
with confidence until the builder fixes it, and `htmlineage sql` refuses
|
|
19
|
+
a report with errors.
|
|
20
|
+
- **Ungrounded.** Claims and numbers that rest on nothing queried: a forecast,
|
|
21
|
+
an opinion, outside knowledge. List each one with its text. They may be
|
|
22
|
+
fine, but the reader should know they are not backed by data.
|
|
23
|
+
- **External.** Data the builder was handed: a CSV, a number someone said.
|
|
24
|
+
Name each one and who it came from (its `given`). A reviewer may want to
|
|
25
|
+
confirm it with that person.
|
|
26
|
+
- **Warnings.** Each is a gap a reader could trip on. The ones that matter
|
|
27
|
+
most: a number in prose with no lineage (C2), a page that shows a different
|
|
28
|
+
value from the graph (C3), and a source without literal dates (C7).
|
|
29
|
+
|
|
30
|
+
## 2. Trace the numbers that carry the report
|
|
31
|
+
|
|
32
|
+
Pick the headline numbers and the ones the main claims rest on. For each, take
|
|
33
|
+
its id from the element's `data-lineage` attribute and run:
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
npx htmlineage sql <report>.html --id <id>
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Read the script against the claim on the page:
|
|
40
|
+
|
|
41
|
+
- **Population.** Does the source cover who the claim is about? A claim about
|
|
42
|
+
all customers over a source filtered to one region is a finding.
|
|
43
|
+
- **Period.** Do the literal dates match the period the page names?
|
|
44
|
+
- **Comparison.** Is a change measured against the base the text says (last
|
|
45
|
+
month, last year, target)?
|
|
46
|
+
- **Definition.** Does the metric match its name: a 30-day rate computed over
|
|
47
|
+
30 days, a share over the right total?
|
|
48
|
+
- **Descriptions.** Does each step's `desc` say what its SQL does? A `desc`
|
|
49
|
+
that promises more than the SQL is a finding.
|
|
50
|
+
|
|
51
|
+
`npx htmlineage sql <report>.html` with no `--id` composes the whole report into
|
|
52
|
+
one script, if the reviewer wants everything at once.
|
|
53
|
+
|
|
54
|
+
## 3. Rerun, if you can and are asked to
|
|
55
|
+
|
|
56
|
+
If you have read access to the warehouse and the reviewer wants the numbers
|
|
57
|
+
reproduced, run the script from step 2 and compare its result with the figure's
|
|
58
|
+
`display` on the page. Run it as it is; do not rewrite it first. Report any
|
|
59
|
+
difference with both values. Because every source has literal dates, the script
|
|
60
|
+
should give the same result as when the report was built, unless the
|
|
61
|
+
underlying data has changed since.
|
|
62
|
+
|
|
63
|
+
Never run anything that writes to the warehouse.
|
|
64
|
+
|
|
65
|
+
## 4. Report back
|
|
66
|
+
|
|
67
|
+
A short list, most serious first:
|
|
68
|
+
|
|
69
|
+
1. Errors, if any.
|
|
70
|
+
2. Findings from step 2 and step 3: the id, what the page says, what the SQL
|
|
71
|
+
does, and why it matters.
|
|
72
|
+
3. Ungrounded claims and external data, so the reader knows what to take on
|
|
73
|
+
trust.
|
|
74
|
+
4. Anything you checked and found sound, briefly. "f1, f3 and f5 trace cleanly
|
|
75
|
+
to the August retention query" is useful to a reviewer.
|
|
76
|
+
|
|
77
|
+
Do not edit the report or its graph. If the builder should change something,
|
|
78
|
+
say what and why. For a person to click through the lineage themselves, point
|
|
79
|
+
them to `npx htmlineage view <report>.html`.
|
package/src/check/rules.js
CHANGED
|
@@ -62,7 +62,9 @@ export function parseDisplayNumber(text) {
|
|
|
62
62
|
* Run the check rules over a graph and its page.
|
|
63
63
|
* @param {unknown} graph as returned by readGraph; may be malformed
|
|
64
64
|
* @param {{ bindings: {id,text,offset}[], prose: {text,node,offset}[] }} page
|
|
65
|
-
* @returns {{ code, severity, node: string|null, field: string|null, message: string }[]}
|
|
65
|
+
* @returns {{ code, severity, node: string|null, field: string|null, message: string, fix: string|null }[]}
|
|
66
|
+
* `message` says what is wrong; `fix`, when there is one, what to change.
|
|
67
|
+
* C7 and C8 list several findings in one message and have no `fix`.
|
|
66
68
|
*/
|
|
67
69
|
export function checkRules(graph, page) {
|
|
68
70
|
const nodes = graph?.nodes;
|
|
@@ -72,13 +74,14 @@ export function checkRules(graph, page) {
|
|
|
72
74
|
const node = (id) => (has(id) && isObject(nodes[id]) ? nodes[id] : null);
|
|
73
75
|
|
|
74
76
|
const issues = [];
|
|
75
|
-
const report = (code, n, field, message) =>
|
|
77
|
+
const report = (code, n, field, message, fix) =>
|
|
76
78
|
issues.push({
|
|
77
79
|
code,
|
|
78
80
|
severity: RULES[code].severity,
|
|
79
81
|
node: n,
|
|
80
82
|
field: field ?? null,
|
|
81
83
|
message,
|
|
84
|
+
fix: fix ?? null,
|
|
82
85
|
});
|
|
83
86
|
|
|
84
87
|
checkC1(nodes, has, node, page, report);
|
|
@@ -99,7 +102,15 @@ function checkC1(nodes, has, node, page, report) {
|
|
|
99
102
|
if (seen.has(id)) continue;
|
|
100
103
|
if (!has(id)) {
|
|
101
104
|
seen.add(id);
|
|
102
|
-
report(
|
|
105
|
+
report(
|
|
106
|
+
"C1",
|
|
107
|
+
id,
|
|
108
|
+
null,
|
|
109
|
+
`data-lineage="${id}" resolves to no node`,
|
|
110
|
+
id.trim() === ""
|
|
111
|
+
? "put a node id in the attribute, or remove the attribute"
|
|
112
|
+
: "add a node with that id to the graph, or correct the id on the page",
|
|
113
|
+
);
|
|
103
114
|
continue;
|
|
104
115
|
}
|
|
105
116
|
const n = node(id);
|
|
@@ -109,7 +120,8 @@ function checkC1(nodes, has, node, page, report) {
|
|
|
109
120
|
"C1",
|
|
110
121
|
id,
|
|
111
122
|
null,
|
|
112
|
-
`\`${id}\` is a source
|
|
123
|
+
`\`${id}\` is a source`,
|
|
124
|
+
"bind a figure over it instead",
|
|
113
125
|
);
|
|
114
126
|
}
|
|
115
127
|
}
|
|
@@ -125,6 +137,7 @@ function checkC1(nodes, has, node, page, report) {
|
|
|
125
137
|
id,
|
|
126
138
|
null,
|
|
127
139
|
`${n.step} is not bound to any element on the page`,
|
|
140
|
+
`add data-lineage="${id}" to the element that shows it`,
|
|
128
141
|
);
|
|
129
142
|
}
|
|
130
143
|
}
|
|
@@ -178,9 +191,23 @@ function checkC3(node, page, report) {
|
|
|
178
191
|
}
|
|
179
192
|
if (bad.length > 0) {
|
|
180
193
|
const quoted = bad.map((t) => `"${t}"`).join(", ");
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
194
|
+
if (display !== null) {
|
|
195
|
+
report(
|
|
196
|
+
"C3",
|
|
197
|
+
id,
|
|
198
|
+
"display",
|
|
199
|
+
`page shows ${quoted} but display is "${display}"`,
|
|
200
|
+
"copy `display` onto the page exactly, or update `value` and `display` if the page is right",
|
|
201
|
+
);
|
|
202
|
+
} else {
|
|
203
|
+
report(
|
|
204
|
+
"C3",
|
|
205
|
+
id,
|
|
206
|
+
"display",
|
|
207
|
+
`page shows ${quoted} but value is ${value}`,
|
|
208
|
+
"correct the page text or `value`, and set `display` to what the page shows",
|
|
209
|
+
);
|
|
210
|
+
}
|
|
184
211
|
}
|
|
185
212
|
}
|
|
186
213
|
}
|
|
@@ -209,12 +236,19 @@ function checkC5(nodes, has, node, report) {
|
|
|
209
236
|
}
|
|
210
237
|
}
|
|
211
238
|
|
|
212
|
-
|
|
239
|
+
// Each problem is its own issue, so each keeps its own fix.
|
|
240
|
+
const problem = (message, fix) => report("C5", id, "from", message, fix);
|
|
213
241
|
|
|
214
242
|
const unknown = n.from.filter((p) => !has(p));
|
|
215
|
-
if (unknown.length >
|
|
216
|
-
|
|
217
|
-
`\`from\` names ${unknown.join(", ")}, which
|
|
243
|
+
if (unknown.length > 1) {
|
|
244
|
+
problem(
|
|
245
|
+
`\`from\` names ${unknown.join(", ")}, which are not nodes`,
|
|
246
|
+
"add those nodes, or remove them from `from`",
|
|
247
|
+
);
|
|
248
|
+
} else if (unknown.length === 1) {
|
|
249
|
+
problem(
|
|
250
|
+
`\`from\` names ${unknown[0]}, which is not a node`,
|
|
251
|
+
"add that node, or remove it from `from`",
|
|
218
252
|
);
|
|
219
253
|
}
|
|
220
254
|
|
|
@@ -226,38 +260,45 @@ function checkC5(nodes, has, node, report) {
|
|
|
226
260
|
if (knownParents.length > 0) {
|
|
227
261
|
let ok;
|
|
228
262
|
let message;
|
|
263
|
+
let fix;
|
|
229
264
|
if (step === "view") {
|
|
230
265
|
ok = knownParents.every(
|
|
231
266
|
(p) => p.step === "source" || p.step === "view",
|
|
232
267
|
);
|
|
233
268
|
message = "a view rests on sources and views only";
|
|
269
|
+
fix = "point `from` at the sources and views it selects from";
|
|
234
270
|
} else if (step === "figure" && exprIds === null) {
|
|
235
271
|
ok = knownParents.every(
|
|
236
272
|
(p) => p.step === "source" || p.step === "view",
|
|
237
273
|
);
|
|
238
274
|
message = "a figure with `sql` rests on sources and views only";
|
|
275
|
+
fix = "to compute it from figures, use `expr` instead of `sql`";
|
|
239
276
|
} else if (step === "figure") {
|
|
240
277
|
ok = knownParents.every((p) => p.step === "figure");
|
|
241
278
|
message = "a figure with `expr` rests on figures only";
|
|
279
|
+
fix = "to pick it from a view, use `sql` instead of `expr`";
|
|
242
280
|
} else {
|
|
243
281
|
ok = knownParents.every((p) => p.step !== "source");
|
|
244
282
|
message = "an insight rests on views, figures and insights only";
|
|
283
|
+
fix = "put a figure over the source and rest the insight on that";
|
|
245
284
|
}
|
|
246
|
-
if (!ok)
|
|
285
|
+
if (!ok) problem(message, fix);
|
|
247
286
|
}
|
|
248
287
|
|
|
249
288
|
if (step === "figure" && exprIds !== null) {
|
|
250
289
|
const missing = exprIds.filter((r) => !n.from.includes(r));
|
|
251
|
-
if (missing.length >
|
|
252
|
-
|
|
253
|
-
`\`expr\` uses ${missing.join(", ")}, which
|
|
290
|
+
if (missing.length > 1) {
|
|
291
|
+
problem(
|
|
292
|
+
`\`expr\` uses ${missing.join(", ")}, which are not in \`from\``,
|
|
293
|
+
"add them to `from`",
|
|
294
|
+
);
|
|
295
|
+
} else if (missing.length === 1) {
|
|
296
|
+
problem(
|
|
297
|
+
`\`expr\` uses ${missing[0]}, which is not in \`from\``,
|
|
298
|
+
"add it to `from`",
|
|
254
299
|
);
|
|
255
300
|
}
|
|
256
301
|
}
|
|
257
|
-
|
|
258
|
-
if (parts.length > 0) {
|
|
259
|
-
report("C5", id, "from", parts.join("; "));
|
|
260
|
-
}
|
|
261
302
|
}
|
|
262
303
|
}
|
|
263
304
|
|
|
@@ -300,6 +341,7 @@ function checkC6(nodes, has, report) {
|
|
|
300
341
|
rotated[0],
|
|
301
342
|
"from",
|
|
302
343
|
`cycle ${[...rotated, rotated[0]].join(" -> ")}`,
|
|
344
|
+
`break the loop: take ${rotated[1] ?? rotated[0]} out of ${rotated[0]}'s \`from\`, or remove another link in it`,
|
|
303
345
|
);
|
|
304
346
|
}
|
|
305
347
|
} else if (!state.has(next)) {
|
package/src/check/summary.js
CHANGED
|
@@ -137,7 +137,7 @@ export function formatSummary(summary, { color = false } = {}) {
|
|
|
137
137
|
node: issue.node ?? "graph",
|
|
138
138
|
tag: `${issue.code} ${severity}`,
|
|
139
139
|
paint: severity === "error" ? RED : YELLOW,
|
|
140
|
-
detail: issue.message,
|
|
140
|
+
detail: issue.fix ? `${issue.message}; ${issue.fix}` : issue.message,
|
|
141
141
|
});
|
|
142
142
|
}
|
|
143
143
|
}
|
package/src/commands/check.js
CHANGED
|
@@ -26,7 +26,10 @@ Options
|
|
|
26
26
|
--no-embed Check only; leave the report file untouched
|
|
27
27
|
--json Print the summary as JSON instead of prose
|
|
28
28
|
--quiet Print nothing; rely on the exit code
|
|
29
|
-
-h, --help Show this help
|
|
29
|
+
-h, --help Show this help
|
|
30
|
+
|
|
31
|
+
What each issue code means and how to fix it:
|
|
32
|
+
https://github.com/htmlineage/htmlineage/blob/main/CHECKS.md`;
|
|
30
33
|
|
|
31
34
|
export const options = {
|
|
32
35
|
graph: { type: "string" },
|
package/src/commands/view.js
CHANGED
|
@@ -11,10 +11,12 @@ export const summary = "Serve the report beside its lineage";
|
|
|
11
11
|
|
|
12
12
|
export const usage = `htmlineage view <report.html> [options]
|
|
13
13
|
|
|
14
|
-
Serves a two-pane page on
|
|
15
|
-
whatever you click on the right.
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
Serves a two-pane page on 127.0.0.1: the report on the left, the lineage of
|
|
15
|
+
whatever you click on the right. It serves the report, its graph and files
|
|
16
|
+
beside the report such as images, never a file whose name starts with a dot
|
|
17
|
+
or any other file outside the report's folder. No database, no credentials.
|
|
18
|
+
The viewer itself makes no request off this machine; the report loads what
|
|
19
|
+
it would load in any browser.
|
|
18
20
|
|
|
19
21
|
Options
|
|
20
22
|
--port <number> Port to serve on (default: the first free port from 7171)
|
package/src/graph/expr.js
CHANGED
|
@@ -21,7 +21,8 @@ export function exprRefs(source, ids) {
|
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* Whether the text can be read at all: brackets and quotes close, and there
|
|
24
|
-
* is no comment. Returns null, or { message, position } (0-based)
|
|
24
|
+
* is no comment. Returns null, or { message, position } (0-based), with a
|
|
25
|
+
* `fix` when there is something to change beyond the obvious.
|
|
25
26
|
*/
|
|
26
27
|
export function exprProblem(source) {
|
|
27
28
|
const open = [];
|
|
@@ -36,7 +37,8 @@ export function exprProblem(source) {
|
|
|
36
37
|
} else if (source.startsWith("--", i) || source.startsWith("/*", i)) {
|
|
37
38
|
const mark = source.slice(i, i + 2);
|
|
38
39
|
return {
|
|
39
|
-
message: `\`${mark}\` starts a comment
|
|
40
|
+
message: `\`${mark}\` starts a comment`,
|
|
41
|
+
fix: "put notes in the node's `label`",
|
|
40
42
|
position: i,
|
|
41
43
|
};
|
|
42
44
|
} else if (source[i] === "(") open.push(i);
|
package/src/graph/schema.js
CHANGED
|
@@ -50,7 +50,7 @@ const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
|
50
50
|
const ISO_TIMESTAMP =
|
|
51
51
|
/^(\d{4}-\d{2}-\d{2})T(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/;
|
|
52
52
|
|
|
53
|
-
const HEADER_FIELDS = new Set([
|
|
53
|
+
export const HEADER_FIELDS = new Set([
|
|
54
54
|
"lineage",
|
|
55
55
|
"report_id",
|
|
56
56
|
"built_at",
|
|
@@ -63,7 +63,7 @@ const HEADER_FIELDS = new Set([
|
|
|
63
63
|
const COMMON_FIELDS = ["step", "transform"];
|
|
64
64
|
const DERIVED_FIELDS = [...COMMON_FIELDS, "from", "ungrounded"];
|
|
65
65
|
|
|
66
|
-
const FIELDS = {
|
|
66
|
+
export const FIELDS = {
|
|
67
67
|
sql: new Set([
|
|
68
68
|
...COMMON_FIELDS,
|
|
69
69
|
"kind",
|
|
@@ -120,18 +120,20 @@ function describe(value) {
|
|
|
120
120
|
|
|
121
121
|
/**
|
|
122
122
|
* Validate a parsed graph. Never throws on bad input; returns a flat list of
|
|
123
|
-
* issues, each `{ code, severity, node, field, message }`. `node` is null
|
|
124
|
-
* header problems
|
|
123
|
+
* issues, each `{ code, severity, node, field, message, fix }`. `node` is null
|
|
124
|
+
* for header problems; `fix` is what to change, or null. A few messages still
|
|
125
|
+
* carry advice of their own, such as the style findings C8 lists. An empty list means the shape is sound.
|
|
125
126
|
*/
|
|
126
127
|
export function validateGraph(graph) {
|
|
127
128
|
const issues = [];
|
|
128
|
-
const report = (code, node, field, message) =>
|
|
129
|
+
const report = (code, node, field, message, fix) =>
|
|
129
130
|
issues.push({
|
|
130
131
|
code,
|
|
131
132
|
severity: RULES[code].severity,
|
|
132
133
|
node,
|
|
133
134
|
field: field ?? null,
|
|
134
135
|
message,
|
|
136
|
+
fix: fix ?? null,
|
|
135
137
|
});
|
|
136
138
|
|
|
137
139
|
if (!isObject(graph)) {
|
|
@@ -201,7 +203,8 @@ function checkNode(id, node, report) {
|
|
|
201
203
|
"S2",
|
|
202
204
|
id,
|
|
203
205
|
null,
|
|
204
|
-
`\`${id}\` is not a valid id
|
|
206
|
+
`\`${id}\` is not a valid id`,
|
|
207
|
+
"use lowercase letters, digits and _, starting with a letter",
|
|
205
208
|
);
|
|
206
209
|
}
|
|
207
210
|
|
|
@@ -226,7 +229,13 @@ function checkNode(id, node, report) {
|
|
|
226
229
|
// they are what makes a node unreadable or uncomposable.
|
|
227
230
|
const need = (field, test, what) => {
|
|
228
231
|
if (node[field] === undefined) {
|
|
229
|
-
report(
|
|
232
|
+
report(
|
|
233
|
+
"S4",
|
|
234
|
+
id,
|
|
235
|
+
field,
|
|
236
|
+
`${node.step} is missing \`${field}\``,
|
|
237
|
+
`add it: ${what}`,
|
|
238
|
+
);
|
|
230
239
|
return false;
|
|
231
240
|
}
|
|
232
241
|
if (!test(node[field])) {
|
|
@@ -265,13 +274,27 @@ function checkNode(id, node, report) {
|
|
|
265
274
|
may("desc", isString, "a string");
|
|
266
275
|
if (node.ungrounded !== true) want("desc", DESC_HINT);
|
|
267
276
|
const problem = isString(node.desc) && descProblem(node.desc);
|
|
268
|
-
if (problem)
|
|
277
|
+
if (problem) {
|
|
278
|
+
report(
|
|
279
|
+
"S9",
|
|
280
|
+
id,
|
|
281
|
+
"desc",
|
|
282
|
+
problem,
|
|
283
|
+
`keep it to one line of ${MAX_DESC} or fewer`,
|
|
284
|
+
);
|
|
285
|
+
}
|
|
269
286
|
}
|
|
270
287
|
|
|
271
288
|
if (allowed) {
|
|
272
289
|
for (const field of Object.keys(node)) {
|
|
273
290
|
if (!allowed.has(field)) {
|
|
274
|
-
|
|
291
|
+
const article = /^[aeiou]/.test(node.step) ? "an" : "a";
|
|
292
|
+
report(
|
|
293
|
+
"S5",
|
|
294
|
+
id,
|
|
295
|
+
field,
|
|
296
|
+
`unknown field \`${field}\` on ${article} ${node.step}`,
|
|
297
|
+
);
|
|
275
298
|
}
|
|
276
299
|
}
|
|
277
300
|
}
|
|
@@ -290,9 +313,8 @@ function checkNode(id, node, report) {
|
|
|
290
313
|
/** Why a `desc` is not one short line, or null. Length wins over a break. */
|
|
291
314
|
function descProblem(desc) {
|
|
292
315
|
const length = [...desc].length;
|
|
293
|
-
|
|
294
|
-
if (
|
|
295
|
-
if (/[\r\n]/.test(desc)) return `\`desc\` has a line break; ${rule}`;
|
|
316
|
+
if (length > MAX_DESC) return `\`desc\` is ${length} characters`;
|
|
317
|
+
if (/[\r\n]/.test(desc)) return "`desc` has a line break";
|
|
296
318
|
return null;
|
|
297
319
|
}
|
|
298
320
|
|
|
@@ -352,7 +374,8 @@ function checkDerived(id, node, need, may, report) {
|
|
|
352
374
|
"C4",
|
|
353
375
|
id,
|
|
354
376
|
"from",
|
|
355
|
-
"`from` is empty
|
|
377
|
+
"`from` is empty",
|
|
378
|
+
"list the parents, or set `ungrounded: true` if there are none",
|
|
356
379
|
);
|
|
357
380
|
}
|
|
358
381
|
}
|
|
@@ -393,14 +416,16 @@ function checkFigure(id, node, may, want, report) {
|
|
|
393
416
|
"C9",
|
|
394
417
|
id,
|
|
395
418
|
null,
|
|
396
|
-
"figure has both `sql` and `expr
|
|
419
|
+
"figure has both `sql` and `expr`",
|
|
420
|
+
"keep exactly one",
|
|
397
421
|
);
|
|
398
422
|
} else if (!hasSql && !hasExpr && node.ungrounded !== true) {
|
|
399
423
|
report(
|
|
400
424
|
"C9",
|
|
401
425
|
id,
|
|
402
426
|
null,
|
|
403
|
-
"figure has neither `sql` nor `expr
|
|
427
|
+
"figure has neither `sql` nor `expr`",
|
|
428
|
+
"add one, or set `ungrounded: true`",
|
|
404
429
|
);
|
|
405
430
|
}
|
|
406
431
|
|
|
@@ -412,6 +437,7 @@ function checkFigure(id, node, may, want, report) {
|
|
|
412
437
|
id,
|
|
413
438
|
"expr",
|
|
414
439
|
`cannot read \`${node.expr}\` at character ${problem.position + 1}: ${problem.message}`,
|
|
440
|
+
problem.fix,
|
|
415
441
|
);
|
|
416
442
|
}
|
|
417
443
|
}
|