functionalscript 0.46.1 → 0.47.0

package/fjs/djs/tokenizer/module.f.mjs

@@ -49,13 +49,13 @@ import {
     unicodeRange,
 } from '../../bnf/module.f.mjs'
 import { keywords } from '../../js/keywords/module.f.mjs'
+import { escapeToCodePoint } from '../../js/string_escape/module.f.mjs'
 import { isKeywordToken, mergeTrivia } from '../../js/tokenizer/module.f.mjs'
 import {
-    asterisk, backspace, ht, lf, ff, cr,
-    quotationMark, solidus, reverseSolidus,
+    asterisk, lf,
+    solidus, reverseSolidus,
     hexDigitValue,
-    latinSmallLetterB, latinSmallLetterF,
-    latinSmallLetterN, latinSmallLetterR, latinSmallLetterT, latinSmallLetterU,
+    latinSmallLetterU,
 } from '../../text/ascii/module.f.mjs'
 import { codePointListToString, stringToCodePointList } from '../../text/utf16/module.f.mjs'
 import { mapUnwrap } from '../../types/nullable/module.f.mjs'
@@ -64,6 +64,97 @@ import { stringifyAsTree } from '../serializer/module.f.mjs'
 import { sort } from '../../types/object/module.f.mjs'
 
 // Builds the single-token grammar that jsGrammar's whole-file `tokens` rule repeats.
+/**
+ * The whitespace characters the grammar's `ws` rule matches.
+ *
+ * The grammar's rule and every downstream check of a trivia tag are built from
+ * this one string: the descent parser emits a range-variant branch's tag as the
+ * matched character itself, so the characters the rule accepts *are* the tags it
+ * produces. Spelling them a second time is what let the two drift apart.
+ *
+ * @type {string}
+ */
+const wsChars = ' \t'
+
+/**
+ * The newline characters the grammar's `newLine` rule matches, the single source
+ * for that rule and its tags exactly as {@link wsChars} is.
+ *
+ * @type {string}
+ */
+const nlChars = '\n\r'
+
+/** @type {ReadonlySet<string>} */
+const wsTags = new Set(wsChars)
+
+/** @type {ReadonlySet<string>} */
+const nlTags = new Set(nlChars)
+
+/**
+ * The operator vocabulary: each key is the tag the descent parser emits for that
+ * branch, and each value the characters it matches.
+ *
+ * At module scope because it captures nothing from `buildToken`, and because
+ * `operatorTags` derives the tag set from these keys rather than re-listing them.
+ */
+const operator = {
+    '.': '.',
+    '=>': '=>',
+    '===': '===',
+    '==': '==',
+    '=': '=',
+    '!==': '!==',
+    '!=': '!=',
+    '!': '!',
+    '>>>=': '>>>=',
+    '>>>': '>>>',
+    '>>=': '>>=',
+    '>>': '>>',
+    '>=': '>=',
+    '>': '>',
+    '<<=': '<<=',
+    '<<': '<<',
+    '<=': '<=',
+    '<': '<',
+    '+=': '+=',
+    '++': '++',
+    '+': '+',
+    '-=': '-=',
+    '--': '--',
+    '-': '-',
+    '**=': '**=',
+    '**': '**',
+    '*=': '*=',
+    '*': '*',
+    '/=': '/=',
+    '/': '/',
+    '%=': '%=',
+    '%': '%',
+    '&&=': '&&=',
+    '&&': '&&',
+    '&=': '&=',
+    '&': '&',
+    '||=': '||=',
+    '||': '||',
+    '|=': '|=',
+    '|': '|',
+    '^=': '^=',
+    '^': '^',
+    '~': '~',
+    '??=': '??=',
+    '??': '??',
+    '?.': '?.',
+    '?': '?',
+    '[': '[',
+    ']': ']',
+    '{': '{',
+    '}': '}',
+    '(': '(',
+    ')': ')',
+    ',': ',',
+    ':': ':'
+}
+
 /** @type {() => Rule} */
 const buildToken = () => {
 
@@ -99,9 +190,9 @@ const buildToken = () => {
 
     const digits = [digit, digits0]
 
-    const ws = set(' \t')
+    const ws = set(wsChars)
 
-    const newLine = set('\n\r')
+    const newLine = set(nlChars)
 
     const idStart = {
         smallLetter: range('az'),
@@ -143,64 +234,6 @@ const buildToken = () => {
 
     const id = [idStart, repeat0Plus(idChar)]
 
-    const operator = {
-        '.': '.',
-        '=>': '=>',
-        '===': '===',
-        '==': '==',
-        '=': '=',
-        '!==': '!==',
-        '!=': '!=',
-        '!': '!',
-        '>>>=': '>>>=',
-        '>>>': '>>>',
-        '>>=': '>>=',
-        '>>': '>>',
-        '>=': '>=',
-        '>': '>',
-        '<<=': '<<=',
-        '<<': '<<',
-        '<=': '<=',
-        '<': '<',
-        '+=': '+=',
-        '++': '++',
-        '+': '+',
-        '-=': '-=',
-        '--': '--',
-        '-': '-',
-        '**=': '**=',
-        '**': '**',
-        '*=': '*=',
-        '*': '*',
-        '/=': '/=',
-        '/': '/',
-        '%=': '%=',
-        '%': '%',
-        '&&=': '&&=',
-        '&&': '&&',
-        '&=': '&=',
-        '&': '&',
-        '||=': '||=',
-        '||': '||',
-        '|=': '|=',
-        '|': '|',
-        '^=': '^=',
-        '^': '^',
-        '~': '~',
-        '??=': '??=',
-        '??': '??',
-        '?.': '?.',
-        '?': '?',
-        '[': '[',
-        ']': ']',
-        '{': '{',
-        '}': '}',
-        '(': '(',
-        ')': ')',
-        ',': ',',
-        ':': ':'
-    }
-
     // Recursive rule: tries end (*/) first at every position so **/  → content(*) + terminator(*/).
     // Falls back to the empty `unterminated` alternative at EOF instead of failing, so `comment`
     // always succeeds and the descent parser never backtracks into matching '/' and '*' as
@@ -295,8 +328,8 @@ const codePointsWithMetadata = path => cp => toArray(flat(stateScan(metadataScan
  * @type {(tag: string) => Nullable<TriviaKind>}
  */
 const triviaKind = tag =>
-    tag === '\n' || tag === '\r' ? 'nl' :
-    tag === ' ' || tag === '\t' ? 'ws' :
+    nlTags.has(tag) ? 'nl' :
+    wsTags.has(tag) ? 'ws' :
     null
 
 /** @type {StateScan<_FlatToken, _TokenScanState, List<_Token>>} */
@@ -325,17 +358,18 @@ const scanFunc = (input, state) => {
     return [null, [stateTag, startMetadata, concat(stateCodePoints)([cp])]]
 }
 
-// All operator tag strings produced by the grammar's operator rule
-const operatorTags = new Set([
-    '.', '=>', '===', '==', '=', '!==', '!=', '!',
-    '>>>=', '>>>', '>>=', '>>', '>=', '>',
-    '<<=', '<<', '<=', '<',
-    '+=', '++', '+', '-=', '--', '-',
-    '**=', '**', '*=', '*', '/=', '/', '%=', '%', // TODO: '/' here causes multi-char line comments (e.g. //ab\n) to fall through as two '/' operator tokens because the oneline rule only consumes one non-newline character (option vs repeat0Plus). Fix the comment rule to use repeat0Plus, or make filterFunc distinguish operator-branch '/' from slash characters inside comments.
-    '&&=', '&&', '&=', '&', '||=', '||', '|=', '|',
-    '^=', '^', '~', '??=', '??', '?.', '?',
-    '[', ']', '{', '}', '(', ')', ',', ':'
-])
+/**
+ * All operator tag strings the grammar's `operator` rule produces, derived from
+ * the rule itself: the descent parser emits a variant branch's key as its tag,
+ * so the keys of {@link operator} *are* the tag set.
+ *
+ * `'/'` is a member because division is an operator; it does not also swallow
+ * the slashes of a comment, since `oneline` consumes the whole rest of the line
+ * with `repeat0Plus` and the comment rule matches before the tag reaches here.
+ *
+ * @type {ReadonlySet<string>}
+ */
+const operatorTags = new Set(Object.keys(operator))
 
 /** @type {(tk: _FlatToken) => boolean} */
 const filterFunc = tk => {
@@ -346,13 +380,11 @@ const filterFunc = tk => {
         case 'string':
         case 'id':
         case 'comment':
-        case '\n':
-        case '\r':
-        case ' ':
-        case '\t':
             return true
         default:
-            return operatorTags.has(tk)
+            // Trivia tags go through `triviaKind` rather than a third copy of
+            // the characters, so they cannot drift from the grammar's rules.
+            return triviaKind(tk) !== null || operatorTags.has(tk)
     }
 }
 
@@ -374,32 +406,17 @@ const unwrapHexDigitValue = mapUnwrap(hexDigitValue)
 const stringDecodeScan = (cp, state) => {
     switch (state.kind) {
         case 'escape': {
+            const codePoint = escapeToCodePoint(cp)
             // The grammar's own `escape` rule (`buildToken`'s `string`) only ever
-            // accepts `"`, `\`, `/`, `b`, `f`, `n`, `r`, `t`, or `u` right after a
+            // accepts one of the eight simple escapes or `u` right after a
             // backslash — any other character fails to parse before a token
             // reaches this scan at all, so narrowing to those nine is provable,
-            // not merely assumed.
-            assert(
-                cp === quotationMark || cp === reverseSolidus || cp === solidus ||
-                cp === latinSmallLetterB || cp === latinSmallLetterF || cp === latinSmallLetterN ||
-                cp === latinSmallLetterR || cp === latinSmallLetterT || cp === latinSmallLetterU,
-                cp)
-            switch (cp) {
-                case quotationMark:  return [[quotationMark],  { kind: 'normal' }]  // \" → "
-                case reverseSolidus: return [[reverseSolidus], { kind: 'normal' }]  // \\ → \
-                case solidus:        return [[solidus],        { kind: 'normal' }]  // \/ → /
-                case latinSmallLetterB: return [[backspace], { kind: 'normal' }]    // \b → backspace (BS)
-                case latinSmallLetterF: return [[ff],        { kind: 'normal' }]    // \f → form feed (FF)
-                case latinSmallLetterN: return [[lf],        { kind: 'normal' }]    // \n → line feed (LF)
-                case latinSmallLetterR: return [[cr],        { kind: 'normal' }]    // \r → carriage return (CR)
-                case latinSmallLetterT: return [[ht],        { kind: 'normal' }]    // \t → horizontal tab (HT)
-                // `\u` is the only case the assertion above leaves. It is a
-                // `default` rather than a `case latinSmallLetterU` because the
-                // ASCII constants are plain `number`s, so the switch can never
-                // be exhaustive to TypeScript and the clause would fall through
-                // into `unicode` below.
-                default: return [null, { kind: 'unicode', acc: 0, count: 0 }]  // \u → start 4 hex digits
-            }
+            // not merely assumed. `u` is the one the table does not answer for:
+            // the four hex digits that follow decide its meaning.
+            assert(codePoint !== null || cp === latinSmallLetterU, cp)
+            return codePoint === null
+                ? [null, { kind: 'unicode', acc: 0, count: 0 }]  // \u → start 4 hex digits
+                : [[codePoint], { kind: 'normal' }]
         }
         case 'unicode': {
             const acc = (state.acc << 4) | unwrapHexDigitValue(cp)

package/fjs/djs/tokenizer/proof.f.mjs

@@ -800,6 +800,23 @@ export const proof = {
             const result = tokenizeString('/* multiline comment *\n * **/')
             if (result !== '[{"kind":"/*","value":" multiline comment *\\n * *"},{"kind":"nl"},{"kind":"eof"}]') { throw result }
         },
+        // `'/'` is an `operatorTags` member for division, so a line comment's own
+        // slashes must be consumed by the comment rule before the tag reaches
+        // `filterFunc`. These pin that: a multi-character line comment stays one
+        // token, inner slashes included, and division still tokenizes as an
+        // operator.
+        () => {
+            const result = tokenizeString('//ab\n')
+            assertEq(result, '[{"kind":"//","value":"ab"},{"kind":"nl"},{"kind":"eof"}]')
+        },
+        () => {
+            const result = tokenizeString('//a//b\n')
+            assertEq(result, '[{"kind":"//","value":"a//b"},{"kind":"nl"},{"kind":"eof"}]')
+        },
+        () => {
+            const result = tokenizeString('a/b')
+            assertEq(result, '[{"kind":"id","value":"a"},{"kind":"/"},{"kind":"id","value":"b"},{"kind":"eof"}]')
+        },
     ],
     metadata: [
         () => {

package/fjs/djs/transpiler/module.f.mjs

@@ -23,8 +23,7 @@ import { concat as pathConcat } from '../../path/module.f.mjs'
 import { parseFromTokens } from '../parser/module.f.mjs'
 import { parse as jsonParse } from '../../media/json/module.f.mjs'
 import { run } from '../ast/module.f.mjs'
-import { pure } from '../../effects/module.f.mjs'
-import { catchStep, foldStep, mapStep, pureError, pureOk, step } from '../../effects/module.f.mjs'
+import { catchStep, foldStep, mapStep, pure, pureError, pureOk, step } from '../../effects/module.f.mjs'
 import { readUtf8File } from '../../effects/node/module.f.mjs'
 
 /**
@@ -65,16 +64,16 @@ const transpileWithImports = path => module => context => {
     const pathsArray = toArray(pathsCombine)
     const contextWithStack = { ...context, stack: { first: path, tail: context.stack } }
     const x0 = foldStep(pureOk(pathsArray), contextWithStack, foldNextModuleOp)
-    return step(
+    return mapStep(
         x0,
         contextWithImports => {
             const args = toArray(listMap(mapDjs(contextWithImports))(pathsCombine))
             const djs = { djs: run(module[1])(args) }
-            return pureOk({
+            return {
                 ...contextWithImports,
                 stack: drop(1)(contextWithImports.stack),
                 complete: setReplace(path)(djs)(contextWithImports.complete),
-            })
+            }
         })
 }
 

package/fjs/djs/transpiler/proof.f.d.mts

@@ -5,6 +5,8 @@ export declare const proof: {
     parseWithIdentifierKeys: () => void;
     parseWithConstIdentifier: () => void;
     parseWithUnaryMinusOperator: () => void;
+    parseAbsolutePath: () => void;
+    parseRelativePathEscapesRoot: () => void;
     parseWithFileNotFoundError: () => void;
     parseWithCycleError: () => void;
 };

package/fjs/djs/transpiler/proof.f.mjs

@@ -59,6 +59,31 @@ export const proof = {
         const s = stringifyAsTree(sort)(result[1])
         assertEq(s, '[-1,2,-3]')
     },
+    // A module named by an absolute path resolves its imports: `transpile`
+    // reaches them through `concat(concat(path)('..'))(importPath)`, which
+    // used to drop the leading `/` and look the import up under the current
+    // directory instead. Here the import also names `..` from the root, so
+    // the answer depends on the root surviving both calls.
+    parseAbsolutePath: () => {
+        const result = run({
+            'lib.f.js': [utf8('export default 8080')],
+            'm.f.js': [utf8('import p from "../lib.f.js"\nexport default p')],
+        })('/m.f.js')
+        assert(result[0] !== 'error', result[1])
+        const s = stringifyAsTree(sort)(result[1])
+        assertEq(s, '8080')
+    },
+    // The control: with no root to clamp it, the same `..` escapes and finds
+    // nothing — which is why the case above is about the root and not about
+    // `..` being ignored.
+    parseRelativePathEscapesRoot: () => {
+        const result = run({
+            'lib.f.js': [utf8('export default 8080')],
+            'm.f.js': [utf8('import p from "../lib.f.js"\nexport default p')],
+        })('m.f.js')
+        assert(result[0] === 'error', result)
+        assertEq(result[1].message, 'file not found', result)
+    },
     parseWithFileNotFoundError: () => {
         const result = run({ a: [utf8('import b from "b"\nexport default b')] })('a')
         assert(result[0] === 'error', result)

package/fjs/edag/amnesia/module.f.d.mts

@@ -0,0 +1,20 @@
+/**
+ * A tree-walking evaluator for `exp`: `vm(context)(e)` returns a primitive
+ * unchanged and dispatches a tagged tuple through one handler per tag.
+ *
+ * It remembers no node values — every incoming edge evaluates its target
+ * again — so it is the Amnesia model of
+ * [execution-models.md](../execution-models.md), for proving semantics and
+ * not for running FunctionalScript. See [README.md](./README.md).
+ *
+ * @module
+ *
+ * @import { Exp, Op1, Properties } from '../types.ts'
+ * @import { OptionLambda, OptionPropertyLambda, PropertyLambda } from '../types.ts'
+ * @import { Context, Map, ExpOp, TagMap } from './types.ts'
+ */
+import type { Exp, Op1 } from '../types.ts';
+import type { Context } from './types.ts';
+export type _Func1 = (c: Context, e: Op1) => unknown;
+export type _Eval = (_: Exp) => unknown;
+export declare const vm: (/**@type {Context}*/ context: Context) => (/**@type{Exp}*/ e: Exp) => unknown;