daftari 3.10.0 → 3.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/README.md +80 -7
- package/dist/access/rbac.d.ts +1 -0
- package/dist/access/rbac.d.ts.map +1 -1
- package/dist/access/rbac.js +6 -0
- package/dist/access/rbac.js.map +1 -1
- package/dist/board/board.test.js +8 -7
- package/dist/board/board.test.js.map +1 -1
- package/dist/board/ledger.test.js +24 -23
- package/dist/board/ledger.test.js.map +1 -1
- package/dist/board/reconcile.test.js +79 -78
- package/dist/board/reconcile.test.js.map +1 -1
- package/dist/board/sources/lint.d.ts.map +1 -1
- package/dist/board/sources/lint.js +6 -2
- package/dist/board/sources/lint.js.map +1 -1
- package/dist/board/sources/lint.test.js +62 -21
- package/dist/board/sources/lint.test.js.map +1 -1
- package/dist/board/sources/staged.test.js +15 -14
- package/dist/board/sources/staged.test.js.map +1 -1
- package/dist/board/sources/staleness.test.js +43 -42
- package/dist/board/sources/staleness.test.js.map +1 -1
- package/dist/board/sources/tension.test.js +15 -14
- package/dist/board/sources/tension.test.js.map +1 -1
- package/dist/board/sources/tier2.test.js +16 -15
- package/dist/board/sources/tier2.test.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +6 -0
- package/dist/cli.js.map +1 -1
- package/dist/consolidate/index.d.ts.map +1 -1
- package/dist/consolidate/index.js +40 -9
- package/dist/consolidate/index.js.map +1 -1
- package/dist/curation/consumes.d.ts +8 -0
- package/dist/curation/consumes.d.ts.map +1 -1
- package/dist/curation/consumes.js +27 -0
- package/dist/curation/consumes.js.map +1 -1
- package/dist/curation/lint-voice.d.ts.map +1 -1
- package/dist/curation/lint-voice.js +3 -1
- package/dist/curation/lint-voice.js.map +1 -1
- package/dist/curation/lint.d.ts +4 -1
- package/dist/curation/lint.d.ts.map +1 -1
- package/dist/curation/lint.js +20 -2
- package/dist/curation/lint.js.map +1 -1
- package/dist/curation/source-refs.d.ts +42 -0
- package/dist/curation/source-refs.d.ts.map +1 -0
- package/dist/curation/source-refs.js +168 -0
- package/dist/curation/source-refs.js.map +1 -0
- package/dist/curation/tension-blast.d.ts.map +1 -1
- package/dist/curation/tension-blast.js +7 -6
- package/dist/curation/tension-blast.js.map +1 -1
- package/dist/curation/tier0.d.ts +0 -1
- package/dist/curation/tier0.d.ts.map +1 -1
- package/dist/curation/tier0.js +24 -23
- package/dist/curation/tier0.js.map +1 -1
- package/dist/curation/vault-docs.d.ts +3 -3
- package/dist/curation/vault-docs.d.ts.map +1 -1
- package/dist/curation/vault-docs.js +12 -5
- package/dist/curation/vault-docs.js.map +1 -1
- package/dist/distill/extract.d.ts.map +1 -1
- package/dist/distill/extract.js +8 -1
- package/dist/distill/extract.js.map +1 -1
- package/dist/distill/propose.d.ts.map +1 -1
- package/dist/distill/propose.js +2 -7
- package/dist/distill/propose.js.map +1 -1
- package/dist/eval/subgraph.d.ts.map +1 -1
- package/dist/eval/subgraph.js +19 -15
- package/dist/eval/subgraph.js.map +1 -1
- package/dist/frontmatter/source-ref.d.ts +24 -0
- package/dist/frontmatter/source-ref.d.ts.map +1 -0
- package/dist/frontmatter/source-ref.js +20 -0
- package/dist/frontmatter/source-ref.js.map +1 -0
- package/dist/schema/diff.d.ts +44 -0
- package/dist/schema/diff.d.ts.map +1 -0
- package/dist/schema/diff.js +143 -0
- package/dist/schema/diff.js.map +1 -0
- package/dist/schema/index.d.ts +2 -0
- package/dist/schema/index.d.ts.map +1 -0
- package/dist/schema/index.js +164 -0
- package/dist/schema/index.js.map +1 -0
- package/dist/schema/infer.d.ts +23 -0
- package/dist/schema/infer.d.ts.map +1 -0
- package/dist/schema/infer.js +167 -0
- package/dist/schema/infer.js.map +1 -0
- package/dist/schema/render.d.ts +12 -0
- package/dist/schema/render.d.ts.map +1 -0
- package/dist/schema/render.js +103 -0
- package/dist/schema/render.js.map +1 -0
- package/dist/schema/scan.d.ts +14 -0
- package/dist/schema/scan.d.ts.map +1 -0
- package/dist/schema/scan.js +92 -0
- package/dist/schema/scan.js.map +1 -0
- package/dist/serve/index.d.ts.map +1 -1
- package/dist/serve/index.js +12 -3
- package/dist/serve/index.js.map +1 -1
- package/dist/sleep/cycle.d.ts +2 -0
- package/dist/sleep/cycle.d.ts.map +1 -1
- package/dist/sleep/cycle.js +95 -16
- package/dist/sleep/cycle.js.map +1 -1
- package/dist/sleep/index.d.ts.map +1 -1
- package/dist/sleep/index.js +8 -7
- package/dist/sleep/index.js.map +1 -1
- package/dist/sleep/report.d.ts.map +1 -1
- package/dist/sleep/report.js +16 -9
- package/dist/sleep/report.js.map +1 -1
- package/dist/sleep/run-ledger.d.ts.map +1 -1
- package/dist/sleep/run-ledger.js +1 -0
- package/dist/sleep/run-ledger.js.map +1 -1
- package/dist/test-utils/require-defined.d.ts +2 -0
- package/dist/test-utils/require-defined.d.ts.map +1 -0
- package/dist/test-utils/require-defined.js +7 -0
- package/dist/test-utils/require-defined.js.map +1 -0
- package/dist/test-utils/require-defined.test.d.ts +2 -0
- package/dist/test-utils/require-defined.test.d.ts.map +1 -0
- package/dist/test-utils/require-defined.test.js +13 -0
- package/dist/test-utils/require-defined.test.js.map +1 -0
- package/dist/tools/board.test.js +20 -17
- package/dist/tools/board.test.js.map +1 -1
- package/dist/tools/curation.d.ts +2 -0
- package/dist/tools/curation.d.ts.map +1 -1
- package/dist/tools/curation.js +32 -3
- package/dist/tools/curation.js.map +1 -1
- package/dist/tools/edge-staleness.d.ts +2 -0
- package/dist/tools/edge-staleness.d.ts.map +1 -1
- package/dist/tools/edge-staleness.js +46 -4
- package/dist/tools/edge-staleness.js.map +1 -1
- package/dist/tools/erase.d.ts +24 -0
- package/dist/tools/erase.d.ts.map +1 -1
- package/dist/tools/erase.js +125 -20
- package/dist/tools/erase.js.map +1 -1
- package/dist/tools/read.d.ts +2 -0
- package/dist/tools/read.d.ts.map +1 -1
- package/dist/tools/read.js +18 -0
- package/dist/tools/read.js.map +1 -1
- package/dist/tools/write.d.ts.map +1 -1
- package/dist/tools/write.js +44 -4
- package/dist/tools/write.js.map +1 -1
- package/dist/utils/config.d.ts +2 -0
- package/dist/utils/config.d.ts.map +1 -1
- package/dist/utils/config.js +27 -0
- package/dist/utils/config.js.map +1 -1
- package/dist/utils/git-erase.d.ts +2 -0
- package/dist/utils/git-erase.d.ts.map +1 -1
- package/dist/utils/git-erase.js +3 -0
- package/dist/utils/git-erase.js.map +1 -1
- package/dist/utils/git.d.ts +1 -0
- package/dist/utils/git.d.ts.map +1 -1
- package/dist/utils/git.js +28 -0
- package/dist/utils/git.js.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { SchemaExtension } from "../utils/config.js";
|
|
2
|
+
import { type ObservedType, type RawFrontmatterDocument } from "./infer.js";
|
|
3
|
+
export interface UndeclaredField {
|
|
4
|
+
field: string;
|
|
5
|
+
occurrences: number;
|
|
6
|
+
prevalence: number;
|
|
7
|
+
types: ObservedType[];
|
|
8
|
+
examples: unknown[];
|
|
9
|
+
distinctValues: number;
|
|
10
|
+
distinctValuesCapped: boolean;
|
|
11
|
+
enumLike: boolean;
|
|
12
|
+
}
|
|
13
|
+
export interface ValueDrift {
|
|
14
|
+
field: string;
|
|
15
|
+
declaredType: SchemaExtension["type"] | "built-in";
|
|
16
|
+
occurrences: number;
|
|
17
|
+
offending: number;
|
|
18
|
+
messages: string[];
|
|
19
|
+
messagesCapped: boolean;
|
|
20
|
+
omittedMessages: number;
|
|
21
|
+
examples: {
|
|
22
|
+
path: string;
|
|
23
|
+
value: unknown;
|
|
24
|
+
}[];
|
|
25
|
+
}
|
|
26
|
+
export interface NearMiss {
|
|
27
|
+
field: string;
|
|
28
|
+
suggestedField: string;
|
|
29
|
+
distance: number;
|
|
30
|
+
occurrences: number;
|
|
31
|
+
}
|
|
32
|
+
export interface SchemaDiff {
|
|
33
|
+
documentCount: number;
|
|
34
|
+
minOccurrences: number;
|
|
35
|
+
undeclared: UndeclaredField[];
|
|
36
|
+
unusedExtensions: Pick<SchemaExtension, "field" | "type" | "required">[];
|
|
37
|
+
valueDrift: ValueDrift[];
|
|
38
|
+
nearMisses: NearMiss[];
|
|
39
|
+
}
|
|
40
|
+
export interface SchemaDiffOptions {
|
|
41
|
+
minOccurrences?: number;
|
|
42
|
+
}
|
|
43
|
+
export declare function diffSchema(documents: RawFrontmatterDocument[], extensions: SchemaExtension[], options?: SchemaDiffOptions): SchemaDiff;
|
|
44
|
+
//# sourceMappingURL=diff.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../../src/schema/diff.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,sBAAsB,EAE5B,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,OAAO,CAAC;IAC9B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CAC9C;AAWD,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,gBAAgB,EAAE,IAAI,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC;IACzE,UAAU,EAAE,UAAU,EAAE,CAAC;IACzB,UAAU,EAAE,QAAQ,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AA0CD,wBAAgB,UAAU,CACxB,SAAS,EAAE,sBAAsB,EAAE,EACnC,UAAU,EAAE,eAAe,EAAE,EAC7B,OAAO,GAAE,iBAAsB,GAC9B,UAAU,CAmIZ"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { validateFrontmatter } from "../frontmatter/schema.js";
|
|
2
|
+
import { BUILTIN_FRONTMATTER_FIELDS } from "../frontmatter/types.js";
|
|
3
|
+
import { compareSchemaText, inferSchema, schemaExampleValue, } from "./infer.js";
|
|
4
|
+
const DRIFT_MESSAGE_LIMIT = 5;
|
|
5
|
+
function driftCategory(message) {
|
|
6
|
+
return message
|
|
7
|
+
.replace(/element \d+/g, "element <index>")
|
|
8
|
+
.replace(/, got .+$/s, ", got <value>")
|
|
9
|
+
.replace(/\(\d+ > \d+\)/g, "(<length> > <limit>)");
|
|
10
|
+
}
|
|
11
|
+
function editDistance(left, right) {
|
|
12
|
+
const previous = Array.from({ length: right.length + 1 }, (_, index) => index);
|
|
13
|
+
for (let i = 1; i <= left.length; i++) {
|
|
14
|
+
const current = [i];
|
|
15
|
+
for (let j = 1; j <= right.length; j++) {
|
|
16
|
+
current[j] = Math.min((current[j - 1] ?? 0) + 1, (previous[j] ?? 0) + 1, (previous[j - 1] ?? 0) + (left[i - 1] === right[j - 1] ? 0 : 1));
|
|
17
|
+
}
|
|
18
|
+
previous.splice(0, previous.length, ...current);
|
|
19
|
+
}
|
|
20
|
+
return previous[right.length] ?? right.length;
|
|
21
|
+
}
|
|
22
|
+
function nearMiss(field, declaredFields) {
|
|
23
|
+
if (field.length < 3)
|
|
24
|
+
return null;
|
|
25
|
+
const normalized = field.toLowerCase();
|
|
26
|
+
const candidates = declaredFields
|
|
27
|
+
.filter((declared) => declared.length >= 3)
|
|
28
|
+
.map((declared) => ({
|
|
29
|
+
suggestedField: declared,
|
|
30
|
+
distance: editDistance(normalized, declared.toLowerCase()),
|
|
31
|
+
}))
|
|
32
|
+
.filter(({ suggestedField, distance }) => distance <= 2 && distance / Math.max(field.length, suggestedField.length) <= 1 / 3)
|
|
33
|
+
.sort((left, right) => left.distance - right.distance ||
|
|
34
|
+
compareSchemaText(left.suggestedField, right.suggestedField));
|
|
35
|
+
return candidates[0] ?? null;
|
|
36
|
+
}
|
|
37
|
+
export function diffSchema(documents, extensions, options = {}) {
|
|
38
|
+
const minOccurrences = Math.max(1, Math.floor(options.minOccurrences ?? 2));
|
|
39
|
+
const inferred = inferSchema(documents);
|
|
40
|
+
const declaredFields = [
|
|
41
|
+
...BUILTIN_FRONTMATTER_FIELDS,
|
|
42
|
+
...extensions.map((extension) => extension.field),
|
|
43
|
+
];
|
|
44
|
+
const declared = new Set(declaredFields);
|
|
45
|
+
const usedExtensions = new Set();
|
|
46
|
+
for (const document of documents) {
|
|
47
|
+
for (const extension of extensions) {
|
|
48
|
+
const value = document.frontmatter[extension.field];
|
|
49
|
+
if (value !== undefined && value !== null)
|
|
50
|
+
usedExtensions.add(extension.field);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
const undeclared = inferred.fields
|
|
54
|
+
.filter((field) => !declared.has(field.field) && field.occurrences >= minOccurrences)
|
|
55
|
+
.map(({ field, occurrences, prevalence, types, examples, distinctValues, distinctValuesCapped, enumLike, }) => ({
|
|
56
|
+
field,
|
|
57
|
+
occurrences,
|
|
58
|
+
prevalence,
|
|
59
|
+
types,
|
|
60
|
+
examples,
|
|
61
|
+
distinctValues,
|
|
62
|
+
distinctValuesCapped,
|
|
63
|
+
enumLike,
|
|
64
|
+
}));
|
|
65
|
+
const unusedExtensions = extensions
|
|
66
|
+
.filter((extension) => !usedExtensions.has(extension.field))
|
|
67
|
+
.map(({ field, type, required }) => ({ field, type, required }))
|
|
68
|
+
.sort((left, right) => compareSchemaText(left.field, right.field));
|
|
69
|
+
const inferredByField = new Map(inferred.fields.map((field) => [field.field, field]));
|
|
70
|
+
const extensionsByField = new Map(extensions.map((extension) => [extension.field, extension]));
|
|
71
|
+
const driftByField = new Map();
|
|
72
|
+
for (const document of documents) {
|
|
73
|
+
const validation = validateFrontmatter(document.frontmatter, extensions);
|
|
74
|
+
const issuesByField = new Map();
|
|
75
|
+
for (const issue of validation.report.issues) {
|
|
76
|
+
if (!declared.has(issue.field) || !Object.hasOwn(document.frontmatter, issue.field))
|
|
77
|
+
continue;
|
|
78
|
+
if (extensionsByField.has(issue.field) &&
|
|
79
|
+
(document.frontmatter[issue.field] === undefined ||
|
|
80
|
+
document.frontmatter[issue.field] === null)) {
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const messages = issuesByField.get(issue.field) ?? [];
|
|
84
|
+
messages.push(issue.message);
|
|
85
|
+
issuesByField.set(issue.field, messages);
|
|
86
|
+
}
|
|
87
|
+
for (const [field, messages] of issuesByField) {
|
|
88
|
+
const accumulator = driftByField.get(field) ?? {
|
|
89
|
+
offending: 0,
|
|
90
|
+
messages: new Map(),
|
|
91
|
+
omittedMessages: 0,
|
|
92
|
+
examples: [],
|
|
93
|
+
};
|
|
94
|
+
accumulator.offending += 1;
|
|
95
|
+
for (const message of messages) {
|
|
96
|
+
const category = String(schemaExampleValue(driftCategory(message)));
|
|
97
|
+
if (accumulator.messages.has(category))
|
|
98
|
+
continue;
|
|
99
|
+
if (accumulator.messages.size < DRIFT_MESSAGE_LIMIT) {
|
|
100
|
+
accumulator.messages.set(category, String(schemaExampleValue(message)));
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
accumulator.omittedMessages += 1;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
if (accumulator.examples.length < 5) {
|
|
107
|
+
accumulator.examples.push({
|
|
108
|
+
path: document.path,
|
|
109
|
+
value: schemaExampleValue(document.frontmatter[field]),
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
driftByField.set(field, accumulator);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const valueDrift = [...driftByField.entries()]
|
|
116
|
+
.sort(([left], [right]) => compareSchemaText(left, right))
|
|
117
|
+
.map(([field, accumulator]) => ({
|
|
118
|
+
field,
|
|
119
|
+
declaredType: extensionsByField.get(field)?.type ?? "built-in",
|
|
120
|
+
occurrences: inferredByField.get(field)?.occurrences ?? 0,
|
|
121
|
+
offending: accumulator.offending,
|
|
122
|
+
messages: [...accumulator.messages.values()].sort(compareSchemaText),
|
|
123
|
+
messagesCapped: accumulator.omittedMessages > 0,
|
|
124
|
+
omittedMessages: accumulator.omittedMessages,
|
|
125
|
+
examples: accumulator.examples.sort((left, right) => compareSchemaText(left.path, right.path)),
|
|
126
|
+
}));
|
|
127
|
+
const nearMisses = inferred.fields
|
|
128
|
+
.filter((field) => !declared.has(field.field))
|
|
129
|
+
.flatMap((field) => {
|
|
130
|
+
const match = nearMiss(field.field, declaredFields);
|
|
131
|
+
return match ? [{ field: field.field, occurrences: field.occurrences, ...match }] : [];
|
|
132
|
+
})
|
|
133
|
+
.sort((left, right) => compareSchemaText(left.field, right.field));
|
|
134
|
+
return {
|
|
135
|
+
documentCount: documents.length,
|
|
136
|
+
minOccurrences,
|
|
137
|
+
undeclared,
|
|
138
|
+
unusedExtensions,
|
|
139
|
+
valueDrift,
|
|
140
|
+
nearMisses,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
//# sourceMappingURL=diff.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diff.js","sourceRoot":"","sources":["../../src/schema/diff.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EACL,iBAAiB,EACjB,WAAW,EAGX,kBAAkB,GACnB,MAAM,YAAY,CAAC;AAwBpB,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO;SACX,OAAO,CAAC,cAAc,EAAE,iBAAiB,CAAC;SAC1C,OAAO,CAAC,YAAY,EAAE,eAAe,CAAC;SACtC,OAAO,CAAC,gBAAgB,EAAE,sBAAsB,CAAC,CAAC;AACvD,CAAC;AAsBD,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa;IAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CACnB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EACzB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EACtB,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAChE,CAAC;QACJ,CAAC;QACD,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC;AAChD,CAAC;AAED,SAAS,QAAQ,CACf,KAAa,EACb,cAAiC;IAEjC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IACvC,MAAM,UAAU,GAAG,cAAc;SAC9B,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;SAC1C,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClB,cAAc,EAAE,QAAQ;QACxB,QAAQ,EAAE,YAAY,CAAC,UAAU,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;KAC3D,CAAC,CAAC;SACF,MAAM,CACL,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC/B,QAAQ,IAAI,CAAC,IAAI,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CACrF;SACA,IAAI,CACH,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ;QAC9B,iBAAiB,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,cAAc,CAAC,CAC/D,CAAC;IACJ,OAAO,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,SAAmC,EACnC,UAA6B,EAC7B,UAA6B,EAAE;IAE/B,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG;QACrB,GAAI,0BAAgD;QACpD,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC;KAClD,CAAC;IACF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAC;IACzC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACpD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBAAE,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM;SAC/B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,IAAI,cAAc,CAAC;SACpF,GAAG,CACF,CAAC,EACC,KAAK,EACL,WAAW,EACX,UAAU,EACV,KAAK,EACL,QAAQ,EACR,cAAc,EACd,oBAAoB,EACpB,QAAQ,GACT,EAAE,EAAE,CAAC,CAAC;QACL,KAAK;QACL,WAAW;QACX,UAAU;QACV,KAAK;QACL,QAAQ;QACR,cAAc;QACd,oBAAoB;QACpB,QAAQ;KACT,CAAC,CACH,CAAC;IAEJ,MAAM,gBAAgB,GAAG,UAAU;SAChC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;SAC3D,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;SAC/D,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAErE,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACtF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAC/F,MAAM,YAAY,GAAG,IAAI,GAAG,EAQzB,CAAC;IACJ,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,UAAU,GAAG,mBAAmB,CAAC,QAAQ,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoB,CAAC;QAClD,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC;gBAAE,SAAS;YAC9F,IACE,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC;gBAClC,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,SAAS;oBAC9C,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,EAC7C,CAAC;gBACD,SAAS;YACX,CAAC;YACD,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACtD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC7B,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC3C,CAAC;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,aAAa,EAAE,CAAC;YAC9C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC7C,SAAS,EAAE,CAAC;gBACZ,QAAQ,EAAE,IAAI,GAAG,EAAkB;gBACnC,eAAe,EAAE,CAAC;gBAClB,QAAQ,EAAE,EAAE;aACb,CAAC;YACF,WAAW,CAAC,SAAS,IAAI,CAAC,CAAC;YAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,kBAAkB,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACpE,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;oBAAE,SAAS;gBACjD,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,GAAG,mBAAmB,EAAE,CAAC;oBACpD,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC1E,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,eAAe,IAAI,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YACD,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACxB,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,KAAK,EAAE,kBAAkB,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;iBACvD,CAAC,CAAC;YACL,CAAC;YACD,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC;SAC3C,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SACzD,GAAG,CACF,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAc,EAAE,CAAC,CAAC;QACrC,KAAK;QACL,YAAY,EAAE,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,UAAU;QAC9D,WAAW,EAAE,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,WAAW,IAAI,CAAC;QACzD,SAAS,EAAE,WAAW,CAAC,SAAS;QAChC,QAAQ,EAAE,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACpE,cAAc,EAAE,WAAW,CAAC,eAAe,GAAG,CAAC;QAC/C,eAAe,EAAE,WAAW,CAAC,eAAe;QAC5C,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAClD,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CACzC;KACF,CAAC,CACH,CAAC;IAEJ,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM;SAC/B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;SAC7C,OAAO,CAAC,CAAC,KAAK,EAAc,EAAE;QAC7B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QACpD,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzF,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAErE,OAAO;QACL,aAAa,EAAE,SAAS,CAAC,MAAM;QAC/B,cAAc;QACd,UAAU;QACV,gBAAgB;QAChB,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/schema/index.ts"],"names":[],"mappings":"AAsGA,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAqF/D"}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { resolve } from "node:path";
|
|
2
|
+
import { err, ok } from "../frontmatter/types.js";
|
|
3
|
+
import { loadConfig } from "../utils/config.js";
|
|
4
|
+
import { diffSchema } from "./diff.js";
|
|
5
|
+
import { inferSchema } from "./infer.js";
|
|
6
|
+
import { renderInferredSchema, renderSchemaDiff } from "./render.js";
|
|
7
|
+
import { normalizeSchemaScope, scanVaultFrontmatter } from "./scan.js";
|
|
8
|
+
const HELP = `daftari schema — infer the vault's de facto frontmatter schema and compare it with declarations.
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
daftari schema infer [--vault <path>] [--scope <folder>] [--json]
|
|
12
|
+
daftari schema diff [--vault <path>] [--scope <folder>] [--min-occurrences <n>] [--json]
|
|
13
|
+
|
|
14
|
+
Commands:
|
|
15
|
+
infer Report each observed frontmatter key, occurrence count, type(s),
|
|
16
|
+
bounded examples, and enum-likeness.
|
|
17
|
+
diff Compare observations with built-ins + schema_extensions. Report
|
|
18
|
+
widely used undeclared fields, unused extensions, invalid observed
|
|
19
|
+
values, and near-miss field names.
|
|
20
|
+
|
|
21
|
+
Options:
|
|
22
|
+
--vault <path> Vault root (default: current directory).
|
|
23
|
+
--scope <folder> Restrict analysis to a vault-relative folder.
|
|
24
|
+
--min-occurrences <n> Minimum use count for undeclared candidates (default: 2).
|
|
25
|
+
--json Emit deterministic JSON instead of Markdown.
|
|
26
|
+
--help, -h Show this help.
|
|
27
|
+
|
|
28
|
+
Read-only and advisory. No markdown, index, or config state is written.
|
|
29
|
+
|
|
30
|
+
Exit codes:
|
|
31
|
+
0 — report produced
|
|
32
|
+
2 — usage or config error
|
|
33
|
+
3 — vault read/runtime error
|
|
34
|
+
`;
|
|
35
|
+
function parseOptions(argv) {
|
|
36
|
+
const mode = argv[0];
|
|
37
|
+
if (mode !== "infer" && mode !== "diff") {
|
|
38
|
+
return err(new Error(`unknown command ${JSON.stringify(mode)} (expected infer or diff)`));
|
|
39
|
+
}
|
|
40
|
+
let vault = ".";
|
|
41
|
+
let scope;
|
|
42
|
+
let json = false;
|
|
43
|
+
let rawMinimum;
|
|
44
|
+
const seen = new Set();
|
|
45
|
+
for (let index = 1; index < argv.length; index++) {
|
|
46
|
+
const argument = argv[index] ?? "";
|
|
47
|
+
if (argument === "--json") {
|
|
48
|
+
if (seen.has("--json"))
|
|
49
|
+
return err(new Error("--json may be passed only once"));
|
|
50
|
+
seen.add("--json");
|
|
51
|
+
json = true;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const equals = argument.indexOf("=");
|
|
55
|
+
const flag = equals === -1 ? argument : argument.slice(0, equals);
|
|
56
|
+
if (!["--vault", "--scope", "--min-occurrences"].includes(flag)) {
|
|
57
|
+
return err(new Error(`unknown argument: ${argument}`));
|
|
58
|
+
}
|
|
59
|
+
if (seen.has(flag))
|
|
60
|
+
return err(new Error(`${flag} may be passed only once`));
|
|
61
|
+
seen.add(flag);
|
|
62
|
+
const inline = equals === -1 ? undefined : argument.slice(equals + 1);
|
|
63
|
+
const value = inline ?? argv[index + 1];
|
|
64
|
+
if (value === undefined || (inline === undefined && value.startsWith("--"))) {
|
|
65
|
+
return err(new Error(`${flag} requires a value`));
|
|
66
|
+
}
|
|
67
|
+
if (value.length === 0) {
|
|
68
|
+
return err(new Error(flag === "--scope" ? "--scope cannot be empty" : `${flag} requires a value`));
|
|
69
|
+
}
|
|
70
|
+
if (inline === undefined)
|
|
71
|
+
index += 1;
|
|
72
|
+
if (flag === "--vault")
|
|
73
|
+
vault = value;
|
|
74
|
+
if (flag === "--scope")
|
|
75
|
+
scope = value;
|
|
76
|
+
if (flag === "--min-occurrences")
|
|
77
|
+
rawMinimum = value;
|
|
78
|
+
}
|
|
79
|
+
if (mode === "infer" && rawMinimum !== undefined) {
|
|
80
|
+
return err(new Error("--min-occurrences is only valid with schema diff"));
|
|
81
|
+
}
|
|
82
|
+
if (rawMinimum !== undefined && !/^[1-9]\d*$/.test(rawMinimum)) {
|
|
83
|
+
return err(new Error("--min-occurrences must be a positive integer"));
|
|
84
|
+
}
|
|
85
|
+
const minOccurrences = Number(rawMinimum ?? "2");
|
|
86
|
+
if (!Number.isSafeInteger(minOccurrences)) {
|
|
87
|
+
return err(new Error("--min-occurrences must be a positive integer"));
|
|
88
|
+
}
|
|
89
|
+
const normalizedScope = normalizeSchemaScope(scope);
|
|
90
|
+
if (!normalizedScope.ok)
|
|
91
|
+
return normalizedScope;
|
|
92
|
+
return ok({ mode, vault: resolve(vault), scope: normalizedScope.value, json, minOccurrences });
|
|
93
|
+
}
|
|
94
|
+
export async function runSchema(argv) {
|
|
95
|
+
if (argv.includes("--help") || argv.includes("-h")) {
|
|
96
|
+
process.stdout.write(HELP);
|
|
97
|
+
return 0;
|
|
98
|
+
}
|
|
99
|
+
if (argv.length === 0) {
|
|
100
|
+
process.stderr.write(HELP);
|
|
101
|
+
return 2;
|
|
102
|
+
}
|
|
103
|
+
const parsed = parseOptions(argv);
|
|
104
|
+
if (!parsed.ok) {
|
|
105
|
+
process.stderr.write(`daftari schema: ${parsed.error.message}\n`);
|
|
106
|
+
return 2;
|
|
107
|
+
}
|
|
108
|
+
const options = parsed.value;
|
|
109
|
+
let extensions = [];
|
|
110
|
+
if (options.mode === "diff") {
|
|
111
|
+
const config = loadConfig(options.vault);
|
|
112
|
+
if (!config.ok) {
|
|
113
|
+
process.stderr.write(`daftari schema: ${config.error.message}\n`);
|
|
114
|
+
return 2;
|
|
115
|
+
}
|
|
116
|
+
extensions = config.value.schemaExtensions;
|
|
117
|
+
}
|
|
118
|
+
const scanned = await scanVaultFrontmatter(options.vault, options.scope);
|
|
119
|
+
if (!scanned.ok) {
|
|
120
|
+
process.stderr.write(`daftari schema: ${scanned.error.message}\n`);
|
|
121
|
+
return 3;
|
|
122
|
+
}
|
|
123
|
+
if (options.mode === "infer") {
|
|
124
|
+
const inferred = inferSchema(scanned.value.documents);
|
|
125
|
+
if (options.json) {
|
|
126
|
+
process.stdout.write(`${JSON.stringify({
|
|
127
|
+
mode: "infer",
|
|
128
|
+
vault: options.vault,
|
|
129
|
+
scope: options.scope ?? null,
|
|
130
|
+
filesScanned: scanned.value.filesScanned,
|
|
131
|
+
documentsAnalyzed: inferred.documentCount,
|
|
132
|
+
scanIssues: scanned.value.issues,
|
|
133
|
+
fields: inferred.fields,
|
|
134
|
+
}, null, 2)}\n`);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
process.stdout.write(renderInferredSchema(options.vault, options.scope, scanned.value, inferred));
|
|
138
|
+
}
|
|
139
|
+
return 0;
|
|
140
|
+
}
|
|
141
|
+
const diff = diffSchema(scanned.value.documents, extensions, {
|
|
142
|
+
minOccurrences: options.minOccurrences,
|
|
143
|
+
});
|
|
144
|
+
if (options.json) {
|
|
145
|
+
process.stdout.write(`${JSON.stringify({
|
|
146
|
+
mode: "diff",
|
|
147
|
+
vault: options.vault,
|
|
148
|
+
scope: options.scope ?? null,
|
|
149
|
+
filesScanned: scanned.value.filesScanned,
|
|
150
|
+
documentsAnalyzed: diff.documentCount,
|
|
151
|
+
scanIssues: scanned.value.issues,
|
|
152
|
+
minOccurrences: diff.minOccurrences,
|
|
153
|
+
undeclared: diff.undeclared,
|
|
154
|
+
unusedExtensions: diff.unusedExtensions,
|
|
155
|
+
valueDrift: diff.valueDrift,
|
|
156
|
+
nearMisses: diff.nearMisses,
|
|
157
|
+
}, null, 2)}\n`);
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
process.stdout.write(renderSchemaDiff(options.vault, options.scope, scanned.value, diff));
|
|
161
|
+
}
|
|
162
|
+
return 0;
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/schema/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AAEvE,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;CA0BZ,CAAC;AAUF,SAAS,YAAY,CAAC,IAAc;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACrB,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACxC,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,CAAC;IAC5F,CAAC;IACD,IAAI,KAAK,GAAG,GAAG,CAAC;IAChB,IAAI,KAAyB,CAAC;IAC9B,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,UAA8B,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC,CAAC;YAChF,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnB,IAAI,GAAG,IAAI,CAAC;YACZ,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QAClE,IAAI,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,mBAAmB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChE,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,IAAI,0BAA0B,CAAC,CAAC,CAAC;QAC7E,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACf,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,MAAM,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACxC,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAC5E,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,IAAI,mBAAmB,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,GAAG,CACR,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,GAAG,IAAI,mBAAmB,CAAC,CACvF,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,KAAK,SAAS;YAAE,KAAK,IAAI,CAAC,CAAC;QACrC,IAAI,IAAI,KAAK,SAAS;YAAE,KAAK,GAAG,KAAK,CAAC;QACtC,IAAI,IAAI,KAAK,SAAS;YAAE,KAAK,GAAG,KAAK,CAAC;QACtC,IAAI,IAAI,KAAK,mBAAmB;YAAE,UAAU,GAAG,KAAK,CAAC;IACvD,CAAC;IAED,IAAI,IAAI,KAAK,OAAO,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACjD,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/D,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE,CAAC;QAC1C,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,eAAe,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IACpD,IAAI,CAAC,eAAe,CAAC,EAAE;QAAE,OAAO,eAAe,CAAC;IAChD,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,eAAe,CAAC,KAAK,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;AACjG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAc;IAC5C,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAClE,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;IAE7B,IAAI,UAAU,GAAsB,EAAE,CAAC;IACvC,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;YAClE,OAAO,CAAC,CAAC;QACX,CAAC;QACD,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC;IAC7C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACzE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QACnE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,IAAI,CAAC,SAAS,CACf;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI;gBAC5B,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,YAAY;gBACxC,iBAAiB,EAAE,QAAQ,CAAC,aAAa;gBACzC,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM;gBAChC,MAAM,EAAE,QAAQ,CAAC,MAAM;aACxB,EACD,IAAI,EACJ,CAAC,CACF,IAAI,CACN,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAC5E,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,UAAU,EAAE;QAC3D,cAAc,EAAE,OAAO,CAAC,cAAc;KACvC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,IAAI,CAAC,SAAS,CACf;YACE,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI;YAC5B,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,YAAY;YACxC,iBAAiB,EAAE,IAAI,CAAC,aAAa;YACrC,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM;YAChC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,EACD,IAAI,EACJ,CAAC,CACF,IAAI,CACN,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;IAC5F,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export interface RawFrontmatterDocument {
|
|
2
|
+
path: string;
|
|
3
|
+
frontmatter: Record<string, unknown>;
|
|
4
|
+
}
|
|
5
|
+
export type ObservedType = "null" | "boolean" | "number" | "date" | "string" | "array" | "object";
|
|
6
|
+
export interface InferredField {
|
|
7
|
+
field: string;
|
|
8
|
+
occurrences: number;
|
|
9
|
+
prevalence: number;
|
|
10
|
+
types: ObservedType[];
|
|
11
|
+
distinctValues: number;
|
|
12
|
+
distinctValuesCapped: boolean;
|
|
13
|
+
examples: unknown[];
|
|
14
|
+
enumLike: boolean;
|
|
15
|
+
}
|
|
16
|
+
export interface InferredSchema {
|
|
17
|
+
documentCount: number;
|
|
18
|
+
fields: InferredField[];
|
|
19
|
+
}
|
|
20
|
+
export declare function compareSchemaText(left: string, right: string): number;
|
|
21
|
+
export declare function schemaExampleValue(value: unknown): unknown;
|
|
22
|
+
export declare function inferSchema(documents: RawFrontmatterDocument[]): InferredSchema;
|
|
23
|
+
//# sourceMappingURL=infer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"infer.d.ts","sourceRoot":"","sources":["../../src/schema/infer.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AAElG,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,OAAO,CAAC;IAC9B,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,aAAa,EAAE,CAAC;CACzB;AAkBD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAErE;AAqFD,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAS1D;AAOD,wBAAgB,WAAW,CAAC,SAAS,EAAE,sBAAsB,EAAE,GAAG,cAAc,CA4D/E"}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
const EXAMPLE_LIMIT = 5;
|
|
3
|
+
const DISTINCT_VALUE_LIMIT = 50;
|
|
4
|
+
const EXAMPLE_CHARACTER_LIMIT = 200;
|
|
5
|
+
const ENUM_DISTINCT_LIMIT = 10;
|
|
6
|
+
const CANONICAL_DEPTH_LIMIT = 20;
|
|
7
|
+
const CANONICAL_NODE_LIMIT = 200;
|
|
8
|
+
const TYPE_ORDER = [
|
|
9
|
+
"null",
|
|
10
|
+
"boolean",
|
|
11
|
+
"number",
|
|
12
|
+
"date",
|
|
13
|
+
"string",
|
|
14
|
+
"array",
|
|
15
|
+
"object",
|
|
16
|
+
];
|
|
17
|
+
export function compareSchemaText(left, right) {
|
|
18
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
19
|
+
}
|
|
20
|
+
function isIsoDate(value) {
|
|
21
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(value))
|
|
22
|
+
return false;
|
|
23
|
+
const parsed = new Date(`${value}T00:00:00.000Z`);
|
|
24
|
+
return !Number.isNaN(parsed.getTime()) && parsed.toISOString().slice(0, 10) === value;
|
|
25
|
+
}
|
|
26
|
+
function observedType(value) {
|
|
27
|
+
if (value === null)
|
|
28
|
+
return "null";
|
|
29
|
+
if (value instanceof Date && !Number.isNaN(value.getTime()))
|
|
30
|
+
return "date";
|
|
31
|
+
if (Array.isArray(value))
|
|
32
|
+
return "array";
|
|
33
|
+
if (typeof value === "object")
|
|
34
|
+
return "object";
|
|
35
|
+
if (typeof value === "string" && isIsoDate(value))
|
|
36
|
+
return "date";
|
|
37
|
+
if (typeof value === "boolean")
|
|
38
|
+
return "boolean";
|
|
39
|
+
if (typeof value === "number")
|
|
40
|
+
return "number";
|
|
41
|
+
return "string";
|
|
42
|
+
}
|
|
43
|
+
function normalizedValue(value) {
|
|
44
|
+
if (value instanceof Date && !Number.isNaN(value.getTime())) {
|
|
45
|
+
return value.toISOString().slice(0, 10);
|
|
46
|
+
}
|
|
47
|
+
return value;
|
|
48
|
+
}
|
|
49
|
+
function canonicalValue(value, state = { ancestors: new WeakSet(), nodes: 0 }, depth = 0) {
|
|
50
|
+
if (state.nodes >= CANONICAL_NODE_LIMIT)
|
|
51
|
+
return "[NodeLimit]";
|
|
52
|
+
state.nodes += 1;
|
|
53
|
+
const normalized = normalizedValue(value);
|
|
54
|
+
if (typeof normalized === "number" && !Number.isFinite(normalized))
|
|
55
|
+
return String(normalized);
|
|
56
|
+
if (typeof normalized === "bigint")
|
|
57
|
+
return `${normalized}n`;
|
|
58
|
+
if (["function", "symbol", "undefined"].includes(typeof normalized)) {
|
|
59
|
+
return `[${typeof normalized}]`;
|
|
60
|
+
}
|
|
61
|
+
if (normalized === null || typeof normalized !== "object")
|
|
62
|
+
return normalized;
|
|
63
|
+
if (state.ancestors.has(normalized))
|
|
64
|
+
return "[Circular]";
|
|
65
|
+
if (depth >= CANONICAL_DEPTH_LIMIT)
|
|
66
|
+
return "[DepthLimit]";
|
|
67
|
+
state.ancestors.add(normalized);
|
|
68
|
+
let canonical;
|
|
69
|
+
if (Array.isArray(normalized)) {
|
|
70
|
+
const items = [];
|
|
71
|
+
for (const item of normalized) {
|
|
72
|
+
if (state.nodes >= CANONICAL_NODE_LIMIT) {
|
|
73
|
+
items.push("[NodeLimit]");
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
76
|
+
items.push(canonicalValue(item, state, depth + 1));
|
|
77
|
+
}
|
|
78
|
+
canonical = items;
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
const record = {};
|
|
82
|
+
const keys = Object.keys(normalized).sort(compareSchemaText);
|
|
83
|
+
for (const key of keys) {
|
|
84
|
+
if (state.nodes >= CANONICAL_NODE_LIMIT) {
|
|
85
|
+
record["[NodeLimit]"] = "[NodeLimit]";
|
|
86
|
+
break;
|
|
87
|
+
}
|
|
88
|
+
record[key] = canonicalValue(normalized[key], state, depth + 1);
|
|
89
|
+
}
|
|
90
|
+
canonical = record;
|
|
91
|
+
}
|
|
92
|
+
state.ancestors.delete(normalized);
|
|
93
|
+
return canonical;
|
|
94
|
+
}
|
|
95
|
+
function serializedValue(value) {
|
|
96
|
+
return JSON.stringify(canonicalValue(value)) ?? String(value);
|
|
97
|
+
}
|
|
98
|
+
function valueKey(value) {
|
|
99
|
+
const type = observedType(value);
|
|
100
|
+
const digest = createHash("sha256").update(serializedValue(value)).digest("hex");
|
|
101
|
+
return `${String(TYPE_ORDER.indexOf(type)).padStart(2, "0")}:${digest}`;
|
|
102
|
+
}
|
|
103
|
+
export function schemaExampleValue(value) {
|
|
104
|
+
const canonical = canonicalValue(value);
|
|
105
|
+
if (typeof canonical === "string" && canonical.length > EXAMPLE_CHARACTER_LIMIT) {
|
|
106
|
+
return `${canonical.slice(0, EXAMPLE_CHARACTER_LIMIT)}…`;
|
|
107
|
+
}
|
|
108
|
+
const serialized = JSON.stringify(canonical) ?? String(canonical);
|
|
109
|
+
return serialized.length > EXAMPLE_CHARACTER_LIMIT
|
|
110
|
+
? `${serialized.slice(0, EXAMPLE_CHARACTER_LIMIT)}…`
|
|
111
|
+
: canonical;
|
|
112
|
+
}
|
|
113
|
+
function exampleSortKey(value) {
|
|
114
|
+
const type = observedType(value);
|
|
115
|
+
return `${String(TYPE_ORDER.indexOf(type)).padStart(2, "0")}:${serializedValue(value)}`;
|
|
116
|
+
}
|
|
117
|
+
export function inferSchema(documents) {
|
|
118
|
+
const byField = new Map();
|
|
119
|
+
for (const document of documents) {
|
|
120
|
+
for (const [field, value] of Object.entries(document.frontmatter)) {
|
|
121
|
+
const accumulator = byField.get(field) ?? {
|
|
122
|
+
occurrences: 0,
|
|
123
|
+
types: new Set(),
|
|
124
|
+
distinct: new Map(),
|
|
125
|
+
distinctValuesCapped: false,
|
|
126
|
+
};
|
|
127
|
+
accumulator.occurrences += 1;
|
|
128
|
+
accumulator.types.add(observedType(value));
|
|
129
|
+
const key = valueKey(value);
|
|
130
|
+
if (accumulator.distinct.has(key)) {
|
|
131
|
+
// Already represented; nothing else to retain.
|
|
132
|
+
}
|
|
133
|
+
else if (accumulator.distinct.size < DISTINCT_VALUE_LIMIT) {
|
|
134
|
+
accumulator.distinct.set(key, schemaExampleValue(value));
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
accumulator.distinctValuesCapped = true;
|
|
138
|
+
}
|
|
139
|
+
byField.set(field, accumulator);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
const fields = [...byField.entries()]
|
|
143
|
+
.sort(([left], [right]) => compareSchemaText(left, right))
|
|
144
|
+
.map(([field, accumulator]) => {
|
|
145
|
+
const types = [...accumulator.types].sort((left, right) => TYPE_ORDER.indexOf(left) - TYPE_ORDER.indexOf(right));
|
|
146
|
+
const examples = [...accumulator.distinct.values()]
|
|
147
|
+
.sort((left, right) => compareSchemaText(exampleSortKey(left), exampleSortKey(right)))
|
|
148
|
+
.slice(0, EXAMPLE_LIMIT);
|
|
149
|
+
const enumLike = !accumulator.distinctValuesCapped &&
|
|
150
|
+
types.length === 1 &&
|
|
151
|
+
["boolean", "number", "string"].includes(types[0] ?? "") &&
|
|
152
|
+
accumulator.distinct.size <= ENUM_DISTINCT_LIMIT &&
|
|
153
|
+
accumulator.distinct.size < accumulator.occurrences;
|
|
154
|
+
return {
|
|
155
|
+
field,
|
|
156
|
+
occurrences: accumulator.occurrences,
|
|
157
|
+
prevalence: documents.length === 0 ? 0 : accumulator.occurrences / documents.length,
|
|
158
|
+
types,
|
|
159
|
+
distinctValues: accumulator.distinct.size,
|
|
160
|
+
distinctValuesCapped: accumulator.distinctValuesCapped,
|
|
161
|
+
examples,
|
|
162
|
+
enumLike,
|
|
163
|
+
};
|
|
164
|
+
});
|
|
165
|
+
return { documentCount: documents.length, fields };
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=infer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"infer.js","sourceRoot":"","sources":["../../src/schema/infer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAyBzC,MAAM,aAAa,GAAG,CAAC,CAAC;AACxB,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AACpC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,UAAU,GAAmB;IACjC,MAAM;IACN,SAAS;IACT,QAAQ;IACR,MAAM;IACN,QAAQ;IACR,OAAO;IACP,QAAQ;CACT,CAAC;AAEF,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,KAAa;IAC3D,OAAO,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,KAAK,gBAAgB,CAAC,CAAC;IAClD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC;AACxF,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAClC,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAAE,OAAO,MAAM,CAAC;IAC3E,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACzC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,SAAS,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IACjE,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACjD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC/C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QAC5D,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAOD,SAAS,cAAc,CACrB,KAAc,EACd,QAAwB,EAAE,SAAS,EAAE,IAAI,OAAO,EAAU,EAAE,KAAK,EAAE,CAAC,EAAE,EACtE,KAAK,GAAG,CAAC;IAET,IAAI,KAAK,CAAC,KAAK,IAAI,oBAAoB;QAAE,OAAO,aAAa,CAAC;IAC9D,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;IACjB,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9F,IAAI,OAAO,UAAU,KAAK,QAAQ;QAAE,OAAO,GAAG,UAAU,GAAG,CAAC;IAC5D,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,OAAO,UAAU,CAAC,EAAE,CAAC;QACpE,OAAO,IAAI,OAAO,UAAU,GAAG,CAAC;IAClC,CAAC;IACD,IAAI,UAAU,KAAK,IAAI,IAAI,OAAO,UAAU,KAAK,QAAQ;QAAE,OAAO,UAAU,CAAC;IAC7E,IAAI,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC;QAAE,OAAO,YAAY,CAAC;IACzD,IAAI,KAAK,IAAI,qBAAqB;QAAE,OAAO,cAAc,CAAC;IAC1D,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAChC,IAAI,SAAkB,CAAC;IACvB,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAc,EAAE,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,KAAK,CAAC,KAAK,IAAI,oBAAoB,EAAE,CAAC;gBACxC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC1B,MAAM;YACR,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,SAAS,GAAG,KAAK,CAAC;IACpB,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC7D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,KAAK,CAAC,KAAK,IAAI,oBAAoB,EAAE,CAAC;gBACxC,MAAM,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC;gBACtC,MAAM;YACR,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAE,UAAsC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAC/F,CAAC;QACD,SAAS,GAAG,MAAM,CAAC;IACrB,CAAC;IACD,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACjF,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,EAAE,CAAC;AAC1E,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;QAChF,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,GAAG,CAAC;IAC3D,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;IAClE,OAAO,UAAU,CAAC,MAAM,GAAG,uBAAuB;QAChD,CAAC,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,GAAG;QACpD,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1F,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,SAAmC;IAC7D,MAAM,OAAO,GAAG,IAAI,GAAG,EAQpB,CAAC;IACJ,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAClE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI;gBACxC,WAAW,EAAE,CAAC;gBACd,KAAK,EAAE,IAAI,GAAG,EAAgB;gBAC9B,QAAQ,EAAE,IAAI,GAAG,EAAmB;gBACpC,oBAAoB,EAAE,KAAK;aAC5B,CAAC;YACF,WAAW,CAAC,WAAW,IAAI,CAAC,CAAC;YAC7B,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;YAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,+CAA+C;YACjD,CAAC;iBAAM,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,GAAG,oBAAoB,EAAE,CAAC;gBAC5D,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,oBAAoB,GAAG,IAAI,CAAC;YAC1C,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAClC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SACzD,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAiB,EAAE;QAC3C,MAAM,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,CACvC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CACtE,CAAC;QACF,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;aAChD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;aACrF,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;QAC3B,MAAM,QAAQ,GACZ,CAAC,WAAW,CAAC,oBAAoB;YACjC,KAAK,CAAC,MAAM,KAAK,CAAC;YAClB,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACxD,WAAW,CAAC,QAAQ,CAAC,IAAI,IAAI,mBAAmB;YAChD,WAAW,CAAC,QAAQ,CAAC,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC;QACtD,OAAO;YACL,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;YACpC,UAAU,EAAE,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,GAAG,SAAS,CAAC,MAAM;YACnF,KAAK;YACL,cAAc,EAAE,WAAW,CAAC,QAAQ,CAAC,IAAI;YACzC,oBAAoB,EAAE,WAAW,CAAC,oBAAoB;YACtD,QAAQ;YACR,QAAQ;SACT,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;AACrD,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { SchemaDiff } from "./diff.js";
|
|
2
|
+
import type { InferredSchema } from "./infer.js";
|
|
3
|
+
import type { SchemaScanIssue } from "./scan.js";
|
|
4
|
+
export declare function renderInferredSchema(vault: string, scope: string | undefined, scan: {
|
|
5
|
+
filesScanned: number;
|
|
6
|
+
issues: SchemaScanIssue[];
|
|
7
|
+
}, schema: InferredSchema): string;
|
|
8
|
+
export declare function renderSchemaDiff(vault: string, scope: string | undefined, scan: {
|
|
9
|
+
filesScanned: number;
|
|
10
|
+
issues: SchemaScanIssue[];
|
|
11
|
+
}, diff: SchemaDiff): string;
|
|
12
|
+
//# sourceMappingURL=render.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"render.d.ts","sourceRoot":"","sources":["../../src/schema/render.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AA4CjD,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,IAAI,EAAE;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,eAAe,EAAE,CAAA;CAAE,EACzD,MAAM,EAAE,cAAc,GACrB,MAAM,CAmBR;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,IAAI,EAAE;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,eAAe,EAAE,CAAA;CAAE,EACzD,IAAI,EAAE,UAAU,GACf,MAAM,CAiER"}
|