functionalscript 0.45.0 → 0.46.1

package/fjs/djs/proof.f.mjs

@@ -1,55 +1,230 @@
 /**
- * @import { Vec } from '../types/bit_vec/types.ts'
+ * @import { Unknown } from './types.ts'
  */
 
+import { exitCode } from '../effects/node/module.f.mjs'
 import { compile } from './module.f.mjs'
+import { transpile } from './transpiler/module.f.mjs'
+import { stringify } from './serializer/module.f.mjs'
 import { virtual, emptyState } from '../effects/node/virtual/module.f.mjs'
 import { utf8, utf8ToString } from '../text/module.f.mjs'
-import { assert, assertEq } from '../asserts/module.f.mjs'
+import { fromEntries, isObject, sort } from '../types/object/module.f.mjs'
+import { assert, assertEq, assertStructurallySame } from '../asserts/module.f.mjs'
 
 /** @type {(root: typeof emptyState.root, path: string) => string} */
 const readOutput = (root, path) => {
     const file = root[path]
     if (!Array.isArray(file) || file.length === 0) { throw `${path} is not a file` }
-    return utf8ToString(/** @type {readonly Vec[]} */ (file)[0])
+    return utf8ToString(file[0])
 }
 
+/** @type {(source: string) => (outputFileName: string) => string} */
+const compileSource = source => outputFileName => {
+    const root = { 'input.f.js': [utf8(source)] }
+    const [state, code] = virtual({ ...emptyState, root })(compile(['input.f.js', outputFileName]))
+    assertEq(exitCode(code), 0, state.stderr)
+    return readOutput(state.root, outputFileName)
+}
+
+const { getPrototypeOf, prototype: objectPrototype } = Object
+
+/** The value every `protoKey` test below denotes. */
+const protoValue = fromEntries([['__proto__', { a: 42 }]])
+
+const sharedArray = [1, 2]
+
+/**
+ * Values the module emitter must be able to write as source that evaluates
+ * back to them. It covers every leaf type, both containers, the shared values
+ * that become a `const`, and the `__proto__` key in both positions — the key
+ * whose obvious spelling evaluates to something else entirely.
+ *
+ * @type {readonly Unknown[]}
+ */
+const roundTripCorpus = [
+    null,
+    true,
+    false,
+    undefined,
+    0,
+    -1.5,
+    42n,
+    'a"b\n\\',
+    [],
+    {},
+    [1, [2, [3, []]]],
+    { a: 1, 'b c': [true, undefined, 3n], d: {} },
+    [sharedArray, sharedArray],
+    { a: 'dup', b: 'dup' },
+    protoValue,
+    fromEntries([['__proto__', 3]]),
+    [protoValue, protoValue],
+    { a: protoValue },
+]
+
 export const proof = {
     tooFewArgs: {
         noArgs: () => {
             const [state, code] = virtual(emptyState)(compile([]))
-            assertEq(code, 1)
+            assertEq(exitCode(code), 1)
             assert(state.stderr.includes('Requires 2 or more arguments'), state.stderr)
         },
         oneArg: () => {
             const [state, code] = virtual(emptyState)(compile(['input.f.js']))
-            assertEq(code, 1)
+            assertEq(exitCode(code), 1)
             assert(state.stderr.includes('Requires 2 or more arguments'), state.stderr)
         },
     },
     success: () => {
         const root = { 'input.f.js': [utf8('export default 42')] }
         const [state, code] = virtual({ ...emptyState, root })(compile(['input.f.js', 'output.f.js']))
-        assertEq(code, 0)
+        assertEq(exitCode(code), 0)
         const content = readOutput(state.root, 'output.f.js')
         assertEq(content, 'export default 42')
     },
     jsonOutput: () => {
         const root = { 'input.f.js': [utf8('export default 42')] }
         const [state, code] = virtual({ ...emptyState, root })(compile(['input.f.js', 'output.json']))
-        assertEq(code, 0)
+        assertEq(exitCode(code), 0)
         const content = readOutput(state.root, 'output.json')
         assertEq(content, '42')
     },
-    fileNotFound: () => {
-        const [state, code] = virtual(emptyState)(compile(['missing.f.js', 'output.f.js']))
-        assertEq(code, 0)
-        assert(state.stderr.includes('file not found'), state.stderr)
+    // An error with no token to point at names the file being compiled, not
+    // `undefined:undefined:undefined`. Each language reports its own missing
+    // file: the module reader and the JSON reader read their inputs
+    // separately.
+    fileNotFound: {
+        module: () => {
+            const [state, code] = virtual(emptyState)(compile(['missing.f.js', 'output.f.js']))
+            assertEq(exitCode(code), 1)
+            assertEq(state.stderr.trim(), 'missing.f.js - error: file not found')
+            assertEq(state.root['output.f.js'], undefined)
+        },
+        json: () => {
+            const [state, code] = virtual(emptyState)(compile(['missing.json', 'output.f.js']))
+            assertEq(exitCode(code), 1)
+            assertEq(state.stderr.trim(), 'missing.json - error: file not found')
+            assertEq(state.root['output.f.js'], undefined)
+        },
     },
     parseError: () => {
         const root = { 'bad.f.js': [utf8('export default @')] }
         const [state, code] = virtual({ ...emptyState, root })(compile(['bad.f.js', 'output.f.js']))
-        assertEq(code, 0)
+        assertEq(exitCode(code), 1)
         assert(state.stderr !== '', 'expected error output')
+        assertEq(state.root['output.f.js'], undefined)
+    },
+    // serialize → evaluate → structurally the same, one test per corpus value.
+    // The emitter is only correct if its output is an input denoting the value
+    // it was given, which no assertion on the text alone can state.
+    roundTrip: roundTripCorpus.map(value => () => {
+        const source = stringify(sort)(value)
+        const root = { 'input.f.js': [utf8(source)] }
+        const [, result] = virtual({ ...emptyState, root })(transpile('input.f.js'))
+        assert(result[0] === 'ok', result[1])
+        assertStructurallySame(result[1], value, source)
+    }),
+    // The `__proto__` key end to end: one value, two output languages, and one
+    // spelling of the key in each (#2480).
+    protoKey: {
+        // The module output uses the computed form, which is also the only
+        // input spelling — so the emitter's output is an input that means the
+        // same value, and compiling it again is the identity.
+        moduleRoundTrip: () => {
+            const source = 'export default {["__proto__"]:{"a":42}}'
+            const output = compileSource(source)('output.f.js')
+            assertEq(output, source)
+            assertEq(compileSource(output)('output.f.js'), source)
+        },
+        // The JSON output keeps the plain key: the computed form is a
+        // JavaScript spelling that no JSON parser accepts.
+        jsonOutput: () => {
+            assertEq(
+                compileSource('export default {["__proto__"]:{"a":42}}')('output.json'),
+                '{"__proto__":{"a":42}}')
+        },
+        // `fjs compile proto.json a.js` — the two languages meeting. The input
+        // is a JSON document, where `"__proto__"` is an ordinary data key, and
+        // the output is a JavaScript module, where only the computed form
+        // denotes one. Each hop uses its own language's spelling of the key.
+        jsonInput: () => {
+            const root = { 'proto.json': [utf8('{"__proto__":5}')] }
+            const [state, code] = virtual({ ...emptyState, root })(compile(['proto.json', 'a.js']))
+            assertEq(exitCode(code), 0, state.stderr)
+            assertEq(readOutput(state.root, 'a.js'), 'export default {["__proto__"]:5}')
+        },
+        // …and back, byte for byte: a JSON document survives the loop
+        // `proto.json → a.js → out.json` with no `["__proto__"]:` artifact,
+        // which no JSON parser would accept.
+        jsonInputRoundTrip: () => {
+            const document = '{"__proto__":{"a":42}}'
+            const root = { 'proto.json': [utf8(document)] }
+            const [state, code] = virtual({ ...emptyState, root })(compile(['proto.json', 'a.js']))
+            assertEq(exitCode(code), 0, state.stderr)
+            const module = readOutput(state.root, 'a.js')
+            assertEq(module, 'export default {["__proto__"]:{"a":42}}')
+            assertEq(compileSource(module)('out.json'), document)
+        },
+        // The extension speaks for the file named on the command line and for
+        // no other: an import is resolved as a FunctionalScript module, and a
+        // JSON document is not one — a statement never begins with a value —
+        // so importing JSON fails until an import can say
+        // `with { type: "json" }` (spec/todo/2140).
+        jsonImportRejected: () => {
+            const root = {
+                'main.f.js': [utf8('import a from "./a.json"\nexport default [a]')],
+                'a.json': [utf8('{"a":42}')],
+            }
+            const [state, code] = virtual({ ...emptyState, root })(compile(['main.f.js', 'out.json']))
+            assertEq(exitCode(code), 1)
+            assert(state.stderr.includes('a.json:1:1 - error: unexpected token'), state.stderr)
+            assertEq(state.root['out.json'], undefined)
+        },
+        // A `.json` input is read as JSON, and an identifier key is no JSON
+        // document's key — so this one fails in the JSON reader, which has no
+        // position to report and is named by its file instead.
+        jsonInputIdKeyRejected: () => {
+            const root = { 'proto.json': [utf8('{__proto__:5}')] }
+            const [state, code] = virtual({ ...emptyState, root })(compile(['proto.json', 'a.js']))
+            assertEq(exitCode(code), 1)
+            assertEq(state.stderr.trim(), 'proto.json - error: unexpected token')
+            assertEq(state.root['a.js'], undefined)
+        },
+        // The `.json` reader is JSON, not DJS with a JSON flag: a bigint is
+        // not JSON, whatever DJS makes of it.
+        jsonInputRejectsDjsExtensions: () => {
+            const root = { 'a.json': [utf8('{"a":1n}')] }
+            const [state, code] = virtual({ ...emptyState, root })(compile(['a.json', 'a.js']))
+            assertEq(exitCode(code), 1)
+            assertEq(state.root['a.js'], undefined)
+        },
+        // The statement behind the textual assertions: the property is an
+        // ordinary own property and the prototype is untouched. A textual test
+        // alone would also pass for a spelling that merely looks right.
+        value: () => {
+            const root = { 'input.f.js': [utf8('export default {["__proto__"]:{"a":42}}')] }
+            const [, result] = virtual({ ...emptyState, root })(transpile('input.f.js'))
+            assert(result[0] === 'ok', result[1])
+            const value = result[1]
+            assert(isObject(value), value)
+            assertStructurallySame(value, protoValue)
+            assertEq(getPrototypeOf(value), objectPrototype)
+        },
+        // The two spellings JavaScript reads as a prototype assignment are
+        // compilation errors, not silently accepted properties.
+        idKeyRejected: () => {
+            const root = { 'input.f.js': [utf8('export default {__proto__:{"a":42}}')] }
+            const [state, code] = virtual({ ...emptyState, root })(compile(['input.f.js', 'output.f.js']))
+            assertEq(exitCode(code), 1)
+            assert(state.stderr.includes('__proto__ requires the computed key form'), state.stderr)
+            assertEq(state.root['output.f.js'], undefined)
+        },
+        stringKeyRejected: () => {
+            const root = { 'input.f.js': [utf8('export default {"__proto__":{"a":42}}')] }
+            const [state, code] = virtual({ ...emptyState, root })(compile(['input.f.js', 'output.f.js']))
+            assertEq(exitCode(code), 1)
+            assert(state.stderr.includes('__proto__ requires the computed key form'), state.stderr)
+            assertEq(state.root['output.f.js'], undefined)
+        },
     },
 }

package/fjs/djs/serializer/module.f.d.mts

@@ -1,30 +1,47 @@
 /**
  * DJS serializer for formatting AST values back to source text.
  *
+ * Two output formats share this one walk over the value: `stringify` emits a
+ * JavaScript module and `stringifyAsTree` a JSON tree. They differ in `const`
+ * hoisting and in how a property key is spelled — see `_KeySerialize`.
+ *
  * @module
  *
- * @import { Unknown, Object } from '../types.ts'
+ * @import { Unknown, Object, _MapEntries } from '../types.ts'
  * @import { Fold } from '../../types/function/operator/types.ts'
- * @import { Entry as ObjectEntry } from '../../types/object/types.ts'
  * @import { List } from '../../types/list/types.ts'
  */
-import type { Unknown } from '../types.ts';
-import type { Entry as ObjectEntry } from '../../types/object/types.ts';
+import type { Unknown, _MapEntries } from '../types.ts';
 import type { List } from '../../types/list/types.ts';
 export declare const undefinedSerialize: string[];
 export type _RefCounter = readonly [number, number];
-export type _MapEntries = (entries: List<ObjectEntry<Unknown>>) => List<ObjectEntry<Unknown>>;
 export type _Refs = ReadonlyMap<Unknown, _RefCounter>;
 export type _GetConstsState = {
     readonly added: ReadonlySet<Unknown>;
     readonly consts: List<Unknown>;
 };
 export type _RefLookup = (value: Unknown) => List<string> | null;
-/** @type {(mapEntries: _MapEntries) => (value: Unknown) => List<string>} */
+export type _KeySerialize = (key: string) => List<string>;
+/**
+ * Serializes a value as a JSON tree — no `const` hoisting, and JSON's key
+ * spelling.
+ *
+ * @type {(mapEntries: _MapEntries) => (value: Unknown) => List<string>}
+ */
 export declare const serializeWithoutConst: (mapEntries: _MapEntries) => (value: Unknown) => List<string>;
-/** @type {(sort: _MapEntries) => (djs: Unknown) => string} */
+/**
+ * Serializes a value as a JavaScript module: a shared value becomes a `const`,
+ * and a `__proto__` key is written in the computed form the language requires.
+ *
+ * @type {(sort: _MapEntries) => (djs: Unknown) => string}
+ */
 export declare const stringify: (sort: _MapEntries) => (djs: Unknown) => string;
-/** @type {(mapEntries: _MapEntries) => (value: Unknown) => string} */
+/**
+ * Serializes a value as a JSON tree: shared values are expanded, and keys keep
+ * JSON's plain spelling.
+ *
+ * @type {(mapEntries: _MapEntries) => (value: Unknown) => string}
+ */
 export declare const stringifyAsTree: (mapEntries: _MapEntries) => (value: Unknown) => string;
 /** @type {(djs: Unknown) => _Refs} */
 export declare const countRefs: (djs: Unknown) => _Refs;

package/fjs/djs/serializer/module.f.mjs

@@ -1,11 +1,14 @@
 /**
  * DJS serializer for formatting AST values back to source text.
  *
+ * Two output formats share this one walk over the value: `stringify` emits a
+ * JavaScript module and `stringifyAsTree` a JSON tree. They differ in `const`
+ * hoisting and in how a property key is spelled — see `_KeySerialize`.
+ *
  * @module
  *
- * @import { Unknown, Object } from '../types.ts'
+ * @import { Unknown, Object, _MapEntries } from '../types.ts'
  * @import { Fold } from '../../types/function/operator/types.ts'
- * @import { Entry as ObjectEntry } from '../../types/object/types.ts'
  * @import { List } from '../../types/list/types.ts'
  */
 
@@ -15,17 +18,13 @@ import { flat, flatMap, map, concat as listConcat } from '../../types/list/modul
 const { entries } = Object
 import { compose, fn } from '../../types/function/module.f.mjs'
 import { serialize as bigintSerialize } from '../../types/bigint/module.f.mjs'
-import { objectWrap, arrayWrap, stringSerialize, numberSerialize, nullSerialize, boolSerialize } from '../../media/json/serializer/module.f.mjs'
+import { objectWrap, arrayWrap, colon, stringSerialize, numberSerialize, nullSerialize, boolSerialize } from '../../media/json/serializer/module.f.mjs'
 import { assertNotNullish } from '../../asserts/module.f.mjs'
 
-const colon = [':']
-
 export const undefinedSerialize = ['undefined']
 
 /** @typedef {readonly [number, number]} _RefCounter */
 
-/** @typedef {(entries: List<ObjectEntry<Unknown>>) => List<ObjectEntry<Unknown>>} _MapEntries */
-
 /** @typedef {ReadonlyMap<Unknown, _RefCounter>} _Refs */
 
 /**
@@ -93,11 +92,43 @@ const entryValue = kv => kv[1]
 /** @type {_RefLookup} */
 const noRef = () => null
 
-/** @type {(refLookup: _RefLookup) => (sort: _MapEntries) => (value: Unknown) => List<string>} */
-const buildSerialize = refLookup => sort => {
+/**
+ * How one output format spells a property key. The two formats disagree about
+ * exactly one key, `__proto__`, so the spelling is a parameter of
+ * `buildSerialize` rather than a property of the shared JSON helper.
+ *
+ * @typedef {(key: string) => List<string>} _KeySerialize
+ */
+
+const protoKey = '__proto__'
+
+/**
+ * JSON spells every key as a quoted string. `"__proto__"` included: `JSON.parse`
+ * has no prototype special case, so the plain spelling already round-trips, and
+ * the computed form below is not JSON at all.
+ *
+ * @type {_KeySerialize}
+ */
+const jsonKeySerialize = stringSerialize
+
+/**
+ * JavaScript reads `{"__proto__": v}` as a prototype assignment, so a module
+ * emitting that spelling would not read back the value it was given. The
+ * computed form is the only spelling whose evaluation reproduces the property,
+ * which makes it a requirement of round-tripping rather than a style choice.
+ * See [spec: the `__proto__` key](../../../spec/README.md#the-__proto__-key).
+ *
+ * @type {_KeySerialize}
+ */
+const jsKeySerialize = key => key === protoKey
+    ? flat([['['], stringSerialize(key), [']']])
+    : stringSerialize(key)
+
+/** @type {(keySerialize: _KeySerialize) => (refLookup: _RefLookup) => (sort: _MapEntries) => (value: Unknown) => List<string>} */
+const buildSerialize = keySerialize => refLookup => sort => {
     /** @type {(kv: readonly [string, Unknown]) => List<string>} */
     const propertySerialize = ([k, v]) => flat([
-        stringSerialize(k),
+        keySerialize(k),
         colon,
         f(v)
     ])
@@ -129,13 +160,18 @@ const buildSerialize = refLookup => sort => {
     return f
 }
 
-/** @type {(mapEntries: _MapEntries) => (value: Unknown) => List<string>} */
-export const serializeWithoutConst = buildSerialize(noRef)
+/**
+ * Serializes a value as a JSON tree — no `const` hoisting, and JSON's key
+ * spelling.
+ *
+ * @type {(mapEntries: _MapEntries) => (value: Unknown) => List<string>}
+ */
+export const serializeWithoutConst = buildSerialize(jsonKeySerialize)(noRef)
 
 /** @type {(sort: _MapEntries) => (refs: _Refs) => (root: Unknown) => (djs: Unknown) => List<string>} */
 const serializeWithConst = sort => refs => {
     const shared = sharedRef(refs)
-    return root => buildSerialize(value => {
+    return root => buildSerialize(jsKeySerialize)(value => {
         if (value === root) { return null }
         const rc = shared(value)
         if (rc !== undefined) { return [`c${rc[0]}`] }
@@ -180,7 +216,12 @@ const addRef = djs => refs => {
     return new Map([...refs, [djs, newCounter]])
 }
 
-/** @type {(sort: _MapEntries) => (djs: Unknown) => string} */
+/**
+ * Serializes a value as a JavaScript module: a shared value becomes a `const`,
+ * and a `__proto__` key is written in the computed form the language requires.
+ *
+ * @type {(sort: _MapEntries) => (djs: Unknown) => string}
+ */
 export const stringify = sort => djs => {
     const refs = countRefs(djs)
     const consts = getConstants(refs)(djs)
@@ -197,7 +238,12 @@ export const stringify = sort => djs => {
     return concat(listConcat(constStrings)(rootStrings))
 }
 
-/** @type {(mapEntries: _MapEntries) => (value: Unknown) => string} */
+/**
+ * Serializes a value as a JSON tree: shared values are expanded, and keys keep
+ * JSON's plain spelling.
+ *
+ * @type {(mapEntries: _MapEntries) => (value: Unknown) => string}
+ */
 export const stringifyAsTree = sort => compose(serializeWithoutConst(sort))(concat)
 
 

package/fjs/djs/serializer/proof.f.d.mts

@@ -12,6 +12,11 @@ export declare const proof: {
         testSort: () => void;
         testIdentity: () => void;
     })[];
+    protoKey: {
+        module: () => void;
+        moduleShared: () => void;
+        json: () => void;
+    };
     stringifyAsTree: ({
         sort: () => void;
         identity: () => void;

package/fjs/djs/serializer/proof.f.mjs

@@ -1,5 +1,5 @@
 import { countRefs, stringify, stringifyAsTree } from './module.f.mjs'
-import { sort } from '../../types/object/module.f.mjs'
+import { fromEntries, sort } from '../../types/object/module.f.mjs'
 import { identity } from '../../types/function/module.f.mjs'
 import { setProperty } from '../../media/json/module.f.mjs'
 import { assertEq } from '../../asserts/module.f.mjs'
@@ -55,6 +55,28 @@ export const proof = {
             },
         }
     ],
+    // The one key the two output formats spell differently. A `__proto__`
+    // property is built with `fromEntries` because an object literal cannot
+    // hold one: `{__proto__: 3}` assigns a prototype instead (#2480).
+    protoKey: {
+        // JavaScript output: only the computed form evaluates back to the
+        // property, so the module emitter must use it.
+        module: () => {
+            const res = stringify(sort)(fromEntries([['__proto__', 3]]))
+            assertEq(res, 'export default {["__proto__"]:3}')
+        },
+        moduleShared: () => {
+            const shared = fromEntries([['__proto__', 3]])
+            const res = stringify(sort)([shared, shared])
+            assertEq(res, 'const c0 = {["__proto__"]:3}\nexport default [c0,c0]')
+        },
+        // JSON output: `JSON.parse` has no prototype special case, so the plain
+        // spelling already round-trips — and the computed form is not JSON.
+        json: () => {
+            const res = stringifyAsTree(sort)(fromEntries([['__proto__', 3]]))
+            assertEq(res, '{"__proto__":3}')
+        },
+    },
     stringifyAsTree: [
         {
             sort: () => {

package/fjs/djs/tokenizer/module.f.d.mts

@@ -3,13 +3,11 @@
  *
  * @module
  *
+ * @import { Ast, AstSequence, AstTag } from '../../bnf/matcher/types.ts'
  * @import {
- *   AstRuleMeta,
- *   AstSequenceMeta,
- *   AstTag,
  *   CodePointMeta,
  *   DescentMatch,
- *   DescentMatchResult
+ *   DescentMatchResult,
  * } from '../../bnf/descent/types.ts'
  * @import { DataRule, Rule } from '../../bnf/types.ts'
  * @import {
@@ -29,8 +27,9 @@
  * @import { CodePoint } from '../../text/utf16/types.ts'
  * @import { StateScan } from '../../types/function/operator/types.ts'
  * @import { List } from '../../types/list/types.ts'
- * @import { Unknown } from '../types.ts'
  * @import { DjsToken, DjsTokenWithMetadata } from './types.ts'
+ * @import { TriviaKind } from '../../js/tokenizer/types.ts'
+ * @import { Nullable } from '../../types/nullable/types.ts'
  */
 import type { CodePointMeta, DescentMatch, DescentMatchResult } from '../../bnf/descent/types.ts';
 import type { Rule } from '../../bnf/types.ts';
@@ -40,6 +39,16 @@ import type { List } from '../../types/list/types.ts';
 import type { DjsTokenWithMetadata } from './types.ts';
 /** @type {() => Rule} */
 export declare const jsGrammar: () => Rule;
+/**
+ * The whole-file matcher together with the name of the rule to start it at.
+ *
+ * `toData` generates rule names, so the entry name belongs to the conversion
+ * rather than to the grammar's spelling and is read back from it here. Building
+ * both from one conversion also keeps the grammar built once per matcher.
+ *
+ * @type {<T>() => readonly [DescentMatch<T>, string]}
+ */
+export declare const jsMatcher: <T>() => readonly [DescentMatch<T>, string];
 /** @type {(m: DescentMatch<unknown>, name: string, cp: readonly CodePoint[]) => DescentMatchResult<unknown>} */
 export declare const descentParserCpOnly: (m: DescentMatch<unknown>, name: string, cp: readonly CodePoint[]) => DescentMatchResult<unknown>;
 export type _Token = [string, TokenMetadata, readonly number[]];
@@ -63,3 +72,6 @@ export type _DjsScanState = {
 };
 /** @type {(input: List<number>) => (path: string) => List<DjsTokenWithMetadata>} */
 export declare const tokenize: (input: List<number>) => (path: string) => List<DjsTokenWithMetadata>;
+export declare const proof: {
+    tagToTokenTrueTag: () => void;
+};