@turri-dev/cli 0.0.0-stage → 0.1.1

package/dist/plain/check.js

@@ -0,0 +1,76 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { basename, extname, join, resolve } from "node:path";
+import { plainFileParser, PLAIN_SOURCE_FILE_EXTENSION } from "./parser.js";
+import { getFrids } from "./spec.js";
+import { resolveTemplateDirs } from "../regenerate/paths.js";
+export function checkPlainFile(file, templateDirs = []) {
+    const path = resolve(file);
+    if (extname(path) !== PLAIN_SOURCE_FILE_EXTENSION) {
+        return {
+            file,
+            ok: false,
+            error: `Not a ${PLAIN_SOURCE_FILE_EXTENSION} file.`,
+            errorType: "UsageError",
+        };
+    }
+    if (!existsSync(path)) {
+        return { file, ok: false, error: "File does not exist.", errorType: "UsageError" };
+    }
+    try {
+        const dirs = resolveTemplateDirs(path, templateDirs);
+        // A module with no functional specs yet is valid - it is how every module starts, and how
+        // an import module always is. It is reported, not failed.
+        const [moduleName, tree, requiredModules] = plainFileParser(basename(path), dirs, { allowNoFunctionalities: true });
+        return {
+            file,
+            ok: true,
+            moduleName,
+            frids: [...getFrids(tree)],
+            requiredModules,
+        };
+    }
+    catch (error) {
+        return {
+            file,
+            ok: false,
+            error: error instanceof Error ? error.message : String(error),
+            errorType: error instanceof Error ? error.name : "Error",
+        };
+    }
+}
+/** Every `.plain` file directly in a directory, or the single file given. */
+export function resolvePlainTargets(target) {
+    const path = resolve(target);
+    if (existsSync(path) && statSync(path).isDirectory()) {
+        return readdirSync(path)
+            .filter((entry) => entry.endsWith(PLAIN_SOURCE_FILE_EXTENSION))
+            .sort()
+            .map((entry) => join(path, entry));
+    }
+    return [path];
+}
+export function checkPlainTargets(target, templateDirs = []) {
+    return resolvePlainTargets(target).map((file) => checkPlainFile(file, templateDirs));
+}
+/** One line per file, plus the parser's message under any that failed. */
+export function formatCheckResults(results) {
+    const lines = [];
+    for (const result of results) {
+        const name = basename(result.file);
+        if (result.ok) {
+            const frids = result.frids ?? [];
+            const requires = result.requiredModules && result.requiredModules.length > 0
+                ? `, requires ${result.requiredModules.join(", ")}`
+                : "";
+            lines.push(frids.length === 0
+                ? `PASS  ${name}  no functional specs yet${requires}`
+                : `PASS  ${name}  ${frids.length} functionalit${frids.length === 1 ? "y" : "ies"} (${frids.join(", ")})${requires}`);
+        }
+        else {
+            lines.push(`FAIL  ${name}  ${result.errorType ?? "Error"}`);
+            for (const line of (result.error ?? "").split("\n"))
+                lines.push(`        ${line}`);
+        }
+    }
+    return lines.join("\n");
+}

package/dist/plain/concepts.js

@@ -0,0 +1,213 @@
+import { PlainSyntaxError } from "./errors.js";
+import { renderToMarkdown } from "./markdown.js";
+import { ACCEPTANCE_TESTS, DEFINITIONS, FUNCTIONAL_REQUIREMENTS, NON_FUNCTIONAL_REQUIREMENTS, TEST_REQUIREMENTS, getSpecificationItems, } from "./spec.js";
+/** Port of concept_utils.py. */
+/** Reserved concept names that must not be redefined in the definitions section. */
+export const DEFAULT_CONCEPTS = new Set([
+    ":ConformanceTests:",
+    ":UnitTests:",
+    ":AcceptanceTest:",
+    ":AcceptanceTests:",
+    ":Implementation:",
+    ":plainDefinitions:",
+    ":plainImplementationReqs:",
+    ":plainFunctionality:",
+    ":plainTestReqs:",
+    ":plainImplementationCode:",
+]);
+const DEFINITION_PATTERN = /-\s(:[^:]+:)(?:,\s*:[^:]+:)*/;
+const CONCEPT_CANDIDATE_PATTERN = /:[^:]+:/g;
+const VALID_CONCEPT_PATTERN = /^:[+\-.0-9A-Z_a-z]+:/;
+const SPEC_CONCEPT_PATTERN = /:[+\-.0-9A-Z_a-z]+:/g;
+export function extractConceptsFromDefinition(text) {
+    const match = DEFINITION_PATTERN.exec(text);
+    if (!match) {
+        return [
+            [],
+            [
+                `Syntax error: Invalid definition specification text: ${text}. Should start with \`- :<concept>:\` ` +
+                    "(where <concept> is any string that does not contain `:`).",
+            ],
+        ];
+    }
+    const candidates = match[0].match(CONCEPT_CANDIDATE_PATTERN) ?? [];
+    const valid = [];
+    const errors = [];
+    for (const candidate of candidates) {
+        if (VALID_CONCEPT_PATTERN.test(candidate)) {
+            valid.push(candidate);
+        }
+        else {
+            errors.push(`Syntax error: Invalid concept: ${candidate}. Should contain only letters, numbers, hyphens and dots.`);
+        }
+    }
+    return [valid, errors];
+}
+export function extractConceptsFromSpecText(text) {
+    return text.match(SPEC_CONCEPT_PATTERN) ?? [];
+}
+export function collectConcepts(plainSource) {
+    const definitions = getSpecificationItems(plainSource, DEFINITIONS);
+    if (definitions === undefined)
+        return [[], []];
+    const concepts = [];
+    const errors = [];
+    for (const definition of definitions) {
+        const [newConcepts, newErrors] = extractConceptsFromDefinition(definition.markdown);
+        errors.push(...newErrors);
+        concepts.push(...newConcepts);
+    }
+    return [concepts, errors];
+}
+function validateConceptsInSpec(spec, concepts, specGroup) {
+    const errors = [];
+    for (const used of extractConceptsFromSpecText(spec.markdown)) {
+        if (!concepts.has(used) && !DEFAULT_CONCEPTS.has(used)) {
+            errors.push(`Syntax error: Concept ${used} is not defined in the definitions. ${specGroup}: ${spec.markdown}`);
+        }
+    }
+    return errors;
+}
+function validateConceptsInSpecGroup(specGroup, plainSource, concepts) {
+    const specs = getSpecificationItems(plainSource, specGroup);
+    if (specs === undefined)
+        return [];
+    return specs.flatMap((spec) => validateConceptsInSpec(spec, concepts, specGroup));
+}
+export function validateConcepts(marshalled) {
+    const errors = [];
+    const concepts = new Set(DEFAULT_CONCEPTS);
+    const [newConcepts, newErrors] = collectConcepts(marshalled);
+    errors.push(...newErrors);
+    const uniqueNewConcepts = new Set(newConcepts);
+    const intersection = [...uniqueNewConcepts].filter((c) => concepts.has(c));
+    if (uniqueNewConcepts.size < newConcepts.length || intersection.length > 0) {
+        const misused = new Set(newConcepts.filter((c) => newConcepts.filter((o) => o === c).length > 1));
+        for (const c of intersection)
+            misused.add(c);
+        errors.push(`Syntax error: Concepts were defined multiple times: ${[...misused].join(", ")} .`);
+    }
+    const treeBasedConcepts = new Set([...DEFAULT_CONCEPTS, ...uniqueNewConcepts]);
+    for (const specGroup of [
+        NON_FUNCTIONAL_REQUIREMENTS,
+        TEST_REQUIREMENTS,
+        FUNCTIONAL_REQUIREMENTS,
+        DEFINITIONS,
+    ]) {
+        errors.push(...validateConceptsInSpecGroup(specGroup, marshalled, treeBasedConcepts));
+    }
+    for (const funcSpec of getSpecificationItems(marshalled, FUNCTIONAL_REQUIREMENTS) ?? []) {
+        for (const test of funcSpec[ACCEPTANCE_TESTS] ?? []) {
+            errors.push(...validateConceptsInSpec(test, treeBasedConcepts, ACCEPTANCE_TESTS));
+        }
+    }
+    return errors;
+}
+export function findConceptDefinitionsInPlainSource(concept, definitions) {
+    const found = [];
+    if (definitions === null)
+        return found;
+    for (const definition of definitions.children) {
+        const rendered = renderToMarkdown(definition);
+        const [definedConcepts] = extractConceptsFromDefinition(rendered);
+        if (!definedConcepts.includes(concept))
+            continue;
+        found.push(definition);
+        const usedConcepts = extractConceptsFromSpecText(rendered).filter((used) => used !== concept && !DEFAULT_CONCEPTS.has(used));
+        for (const used of usedConcepts) {
+            found.push(...findConceptDefinitionsInPlainSource(used, definitions));
+        }
+    }
+    return found;
+}
+function buildAdjacencyList(definitions) {
+    const adjacency = new Map();
+    const conceptDefinitions = new Map();
+    for (const definition of definitions) {
+        const text = definition.markdown;
+        const [newConcepts] = extractConceptsFromDefinition(text);
+        const usedConcepts = new Set(extractConceptsFromSpecText(text).filter((used) => !newConcepts.includes(used) && !DEFAULT_CONCEPTS.has(used)));
+        for (const newConcept of newConcepts) {
+            if (!adjacency.has(newConcept))
+                adjacency.set(newConcept, []);
+        }
+        for (const used of usedConcepts) {
+            if (!adjacency.has(used))
+                adjacency.set(used, []);
+            adjacency.get(used).push(...newConcepts);
+        }
+        for (const newConcept of newConcepts) {
+            conceptDefinitions.set(newConcept, definition);
+        }
+    }
+    return [adjacency, conceptDefinitions];
+}
+/**
+ * networkx's topological_sort, reproduced.
+ *
+ * It flattens `topological_generations`, which is Kahn's algorithm processed one
+ * generation at a time with nodes visited in graph insertion order. The ordering of
+ * independent concepts therefore depends on insertion order, and the Python tests
+ * assert that exact order - a generic topological sort would pass the dependency
+ * constraints but produce a different sequence.
+ */
+function topologicalSort(successors) {
+    const inDegree = new Map();
+    for (const node of successors.keys())
+        inDegree.set(node, 0);
+    for (const [, targets] of successors) {
+        for (const target of targets)
+            inDegree.set(target, (inDegree.get(target) ?? 0) + 1);
+    }
+    let generation = [...successors.keys()].filter((node) => (inDegree.get(node) ?? 0) === 0);
+    const order = [];
+    const remaining = new Map(inDegree);
+    for (const node of generation)
+        remaining.delete(node);
+    while (generation.length > 0) {
+        const next = [];
+        for (const node of generation) {
+            order.push(node);
+            for (const child of successors.get(node) ?? []) {
+                const degree = (remaining.get(child) ?? 0) - 1;
+                remaining.set(child, degree);
+                if (degree === 0) {
+                    next.push(child);
+                    remaining.delete(child);
+                }
+            }
+        }
+        generation = next;
+    }
+    if (remaining.size > 0)
+        return { order, cycle: [...remaining.keys()] };
+    return { order, cycle: null };
+}
+export function sortDefinitions(definitions) {
+    if (definitions.length <= 1)
+        return;
+    const [adjacency, conceptDefinitions] = buildAdjacencyList(definitions);
+    const { order, cycle } = topologicalSort(adjacency);
+    if (cycle !== null) {
+        let msg = "Found cycles in the concept graph. Cycles are not allowed.";
+        const cyclicDefinitions = cycle
+            .slice(0, 2)
+            .map((concept) => conceptDefinitions.get(concept)?.markdown)
+            .filter((markdown) => markdown !== undefined);
+        msg += "Cyclic definitons:\n";
+        msg += cyclicDefinitions.join("\n");
+        msg += "\n";
+        throw new PlainSyntaxError(`Plain syntax error: ${msg}`);
+    }
+    if (order.length === 0)
+        return;
+    const definitionOrder = new Map();
+    order.forEach((concept, index) => {
+        // Python keys this map by definition and lets later concepts overwrite, so a
+        // definition that introduces several concepts takes the index of its last one.
+        const definition = conceptDefinitions.get(concept);
+        if (definition !== undefined)
+            definitionOrder.set(definition, index);
+    });
+    definitions.sort((a, b) => (definitionOrder.get(a) ?? 0) - (definitionOrder.get(b) ?? 0));
+}

package/dist/plain/errors.js

@@ -0,0 +1,43 @@
+/** Ports of the exception classes in plain2code_exceptions.py that this parser raises. */
+export class PlainSyntaxError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "PlainSyntaxError";
+    }
+}
+export class UnsupportedBase64Content extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "UnsupportedBase64Content";
+    }
+}
+export class ModuleDoesNotExistError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ModuleDoesNotExistError";
+    }
+}
+export class MissingFunctionalitiesError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "MissingFunctionalitiesError";
+    }
+}
+export class ImportedModuleWithFunctionalitiesError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ImportedModuleWithFunctionalitiesError";
+    }
+}
+export class InvalidFridArgument extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "InvalidFridArgument";
+    }
+}
+export class InvalidLiquidVariableName extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "InvalidLiquidVariableName";
+    }
+}

package/dist/plain/files.js

@@ -0,0 +1,57 @@
+import { readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { createEnvironment } from "./liquid.js";
+/** Ports of the file_utils.py and plain2code_utils.py helpers this parser needs. */
+export function openFrom(dirs, fileName) {
+    for (const dir of dirs) {
+        const fullPath = join(dir, fileName);
+        try {
+            if (!statSync(fullPath).isFile())
+                continue;
+        }
+        catch {
+            continue;
+        }
+        return readFileSync(fullPath, "utf8");
+    }
+    return null;
+}
+const MAX_BASE64_BLOB_LENGTH = 8192;
+const BASE64_BLOB_PATTERN = new RegExp(`(?:data:[\\w.+-]+/[\\w.+-]+;base64,)?[A-Za-z0-9+/_-]{${MAX_BASE64_BLOB_LENGTH},}={0,2}`);
+/** First contiguous base64 blob at or above the threshold, or null. */
+export function findLargeBase64Blob(text) {
+    const match = BASE64_BLOB_PATTERN.exec(text);
+    return match ? match[0] : null;
+}
+/**
+ * Render once against the filesystem to discover which templates the source pulls in.
+ *
+ * The rendered text is discarded by the caller; what matters is the map of template
+ * name to raw source, which is what the second (real) render is given so that it does
+ * not touch the filesystem again.
+ */
+export function getLoadedTemplates(templateDirs, plainSource) {
+    const loadedTemplates = {};
+    const engine = createEnvironment({
+        strictVariables: true,
+        loadTemplateSource: (name) => {
+            const source = openFrom(templateDirs, name);
+            if (source === null)
+                return null;
+            loadedTemplates[name] = source;
+            return source;
+        },
+    });
+    let rendered;
+    try {
+        rendered = engine.parseAndRenderSync(plainSource, {});
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (/undefined variable/i.test(message)) {
+            throw new Error(`Undefined liquid variable: ${message}`);
+        }
+        throw error;
+    }
+    return [rendered, loadedTemplates];
+}

package/dist/plain/highlight.js

@@ -0,0 +1,195 @@
+import { readFileSync } from "node:fs";
+const compile = (source) => new RegExp(source, "dg");
+const namesOf = (captures) => captures === undefined ? undefined : Object.fromEntries(Object.entries(captures).map(([group, capture]) => [group, capture.name]));
+function readGrammar(raw) {
+    const repository = new Map();
+    const root = [];
+    const read = (rules, into, seen) => {
+        for (const rule of rules) {
+            if (rule.include !== undefined) {
+                // Only the repository's: plyn's grammar includes nothing else, and a rule that includes itself is followed once.
+                if (!rule.include.startsWith("#"))
+                    continue;
+                const key = rule.include.slice(1);
+                if (seen.has(key))
+                    continue;
+                into.push(...repositoryRules(key, seen));
+                continue;
+            }
+            if (rule.match !== undefined) {
+                into.push({ kind: "match", ...(rule.name !== undefined ? { name: rule.name } : {}), re: compile(rule.match), ...(rule.captures !== undefined ? { captures: namesOf(rule.captures) } : {}) });
+            }
+            else if (rule.begin !== undefined && rule.end !== undefined) {
+                const inner = [];
+                read(rule.patterns ?? [], inner, seen);
+                into.push({
+                    kind: "block",
+                    ...(rule.name !== undefined ? { name: rule.name } : {}),
+                    begin: compile(rule.begin),
+                    end: compile(rule.end),
+                    patterns: inner,
+                    ...(rule.beginCaptures ?? rule.captures ? { beginCaptures: namesOf(rule.beginCaptures ?? rule.captures) } : {}),
+                    ...(rule.endCaptures ?? rule.captures ? { endCaptures: namesOf(rule.endCaptures ?? rule.captures) } : {}),
+                });
+            }
+            else if (rule.patterns !== undefined) {
+                read(rule.patterns, into, seen);
+            }
+        }
+    };
+    const repositoryRules = (key, seen) => {
+        const known = repository.get(key);
+        if (known !== undefined)
+            return known;
+        const entry = raw.repository?.[key];
+        if (entry === undefined)
+            return [];
+        const rules = [];
+        repository.set(key, rules);
+        read([entry], rules, new Set([...seen, key]));
+        return rules;
+    };
+    read(raw.patterns, root, new Set());
+    return root;
+}
+let grammar;
+/** plyn's grammar, read once - or null when it cannot be. */
+function plainGrammar() {
+    if (grammar !== undefined)
+        return grammar;
+    try {
+        grammar = readGrammar(JSON.parse(readFileSync(new URL("./plain.tmLanguage.json", import.meta.url), "utf8")));
+    }
+    catch {
+        grammar = null;
+    }
+    return grammar;
+}
+/** The pieces of a match: its capture groups under their scopes, the rest under the rule's. */
+function piecesOf(match, scopes, name, captures) {
+    const own = name !== undefined ? [...scopes, name] : scopes;
+    const [start, end] = match.indices[0];
+    const groups = Object.entries(captures ?? {})
+        .map(([group, scope]) => ({ scope, at: match.indices[Number(group)] }))
+        .filter((group) => group.at !== undefined && group.at[1] > group.at[0])
+        .sort((a, b) => a.at[0] - b.at[0]);
+    const whole = match.input;
+    const tokens = [];
+    let at = start;
+    for (const group of groups) {
+        if (group.at[0] < at)
+            continue;
+        if (group.at[0] > at)
+            tokens.push({ text: whole.slice(at, group.at[0]), scopes: own });
+        tokens.push({ text: whole.slice(group.at[0], group.at[1]), scopes: group.scope !== undefined ? [...own, group.scope] : own });
+        at = group.at[1];
+    }
+    if (end > at)
+        tokens.push({ text: whole.slice(at, end), scopes: own });
+    return tokens;
+}
+/**
+ * One line of ***plain, highlighted: its tokens, and the state the next line starts from - the blocks
+ * still open at its end.
+ */
+export function highlightLine(line, state = []) {
+    const rules = plainGrammar();
+    if (rules === null)
+        return { tokens: line === "" ? [] : [{ text: line, scopes: [] }], state };
+    const stack = [...state];
+    const tokens = [];
+    const scopesNow = () => stack.map((block) => block.name).filter((name) => name !== undefined);
+    const push = (text, scopes) => {
+        if (text === "")
+            return;
+        const last = tokens.at(-1);
+        if (last !== undefined && last.scopes.join(" ") === scopes.join(" "))
+            last.text += text;
+        else
+            tokens.push({ text, scopes });
+    };
+    let at = 0;
+    // A rule that matches nothing at a place moves the place on by one: every line ends.
+    while (at <= line.length) {
+        const top = stack.at(-1);
+        const candidates = top?.patterns ?? rules;
+        let best;
+        const tryRe = (re, rule) => {
+            re.lastIndex = at;
+            const match = re.exec(line);
+            if (match === null)
+                return;
+            // The first to match earliest wins; the end of a block before the rules inside it.
+            if (best === undefined || match.index < best.match.index)
+                best = { match, rule };
+        };
+        if (top !== undefined)
+            tryRe(top.end, "end");
+        for (const rule of candidates)
+            tryRe(rule.kind === "match" ? rule.re : rule.begin, rule);
+        if (best === undefined) {
+            push(line.slice(at), scopesNow());
+            break;
+        }
+        const { match, rule } = best;
+        push(line.slice(at, match.index), scopesNow());
+        const scopes = scopesNow();
+        if (rule === "end") {
+            const block = stack.pop();
+            for (const token of piecesOf(match, scopes.slice(0, block.name !== undefined ? -1 : undefined), block.name, block.endCaptures))
+                push(token.text, token.scopes);
+        }
+        else if (rule.kind === "match") {
+            for (const token of piecesOf(match, scopes, rule.name, rule.captures))
+                push(token.text, token.scopes);
+        }
+        else {
+            for (const token of piecesOf(match, scopes, rule.name, rule.beginCaptures))
+                push(token.text, token.scopes);
+            stack.push(rule);
+        }
+        const next = match.index + match[0].length;
+        if (next === at && match[0].length === 0) {
+            // Nothing matched here: take one character as it is, and go on.
+            push(line.slice(at, at + 1), scopesNow());
+            at += 1;
+        }
+        else
+            at = next;
+        if (at >= line.length)
+            break;
+    }
+    return { tokens, state: stack };
+}
+/** Lines of ***plain, highlighted one after another - a block opened on one carried to the next. */
+export function highlightLines(lines) {
+    let state = [];
+    return lines.map((line) => {
+        const result = highlightLine(line, state);
+        state = result.state;
+        return result.tokens;
+    });
+}
+/** A scope's role, by what it is - the most specific first. None: the text around it. */
+const ROLES = [
+    ["comment", "comment"],
+    ["keyword", "keyword"],
+    ["entity.name.function", "section"],
+    ["entity.name.type", "module"],
+    ["entity.name.module", "module"],
+    ["entity.punctuation", "bullet"],
+    ["punctuation", "bullet"],
+    ["variable", "concept"],
+    // A link is plyn's `string.quoted.other.plain`; its quotes are `string.quoted.double`, `...other.backtick` and the like.
+    ["string.quoted.other.plain", "link"],
+    ["string", "literal"],
+];
+/** A token's role: its innermost scope that has one. */
+export function roleOf(scopes) {
+    for (const scope of [...scopes].reverse()) {
+        const found = ROLES.find(([prefix]) => scope === prefix || scope.startsWith(`${prefix}.`));
+        if (found !== undefined)
+            return found[1];
+    }
+    return undefined;
+}

package/dist/plain/index.js

@@ -0,0 +1,15 @@
+/**
+ * The ***plain*** parser: a TypeScript port of the reference implementation.
+ *
+ * Mirrors plain_file.py, plain_spec.py, concept_utils.py and the file_utils.py
+ * helpers they depend on. The entry points are `plainFileParser` (a .plain file to a
+ * marshalled tree) and `iterateFunctionalSpecs` (the tree to one rendering context per
+ * functionality).
+ */
+export * from "./errors.js";
+export * from "./markdown.js";
+export * from "./concepts.js";
+export * from "./files.js";
+export * from "./liquid.js";
+export * from "./parser.js";
+export * from "./spec.js";