@lowdefy/build 0.0.0-experimental-20251203202233 → 0.0.0-experimental-20260112140412

package/dist/build/buildRefs/evaluateBuildOperators.js

@@ -12,8 +12,14 @@
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-*/ async function evaluateBuildOperators({ context, input, refDef }) {
-    const { output, errors } = context.operatorsParser.parse({
+*/ import { BuildParser } from '@lowdefy/operators';
+import operators from '@lowdefy/operators-js/operators/build';
+async function evaluateBuildOperators({ context, input, refDef }) {
+    const operatorsParser = new BuildParser({
+        env: process.env,
+        operators
+    });
+    const { output, errors } = operatorsParser.parse({
         input,
         location: refDef.path ?? refDef.resolver,
         operatorPrefix: '_build.'

package/dist/build/buildRefs/getConfigFile.js

@@ -21,7 +21,8 @@ async function getConfigFile({ context, refDef, referencedFrom }) {
     }
     const content = await context.readConfigFile(refDef.path);
     if (content === null) {
-        throw new Error(`Tried to reference file "${refDef.path}" from "${referencedFrom}", but file does not exist.`);
+        const lineInfo = refDef.lineNumber ? `:${refDef.lineNumber}` : '';
+        throw new Error(`Tried to reference file "${refDef.path}" from "${referencedFrom}${lineInfo}", but file does not exist.`);
     }
     return content;
 }

package/dist/build/buildRefs/getRefContent.js

@@ -12,47 +12,33 @@
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-*/ import { getFileExtension } from '@lowdefy/node-utils';
-import getConfigFile from './getConfigFile.js';
+*/ import getConfigFile from './getConfigFile.js';
 import parseRefContent from './parseRefContent.js';
 import runRefResolver from './runRefResolver.js';
-function getCacheKey(refDef) {
-    const { path, vars } = refDef;
-    // For nunjucks files, vars affect the output, so include them in the cache key
-    const ext = getFileExtension(path);
-    if (ext === 'njk' && vars) {
-        return `${path}::${JSON.stringify(vars)}`;
-    }
-    return path;
-}
 async function getRefContent({ context, refDef, referencedFrom }) {
-    // Skip caching for resolver-based refs (dynamic content)
-    if (refDef.resolver || context.refResolver) {
-        const content = await runRefResolver({
+    let content;
+    if (refDef.path === 'lowdefy.yaml' || refDef.path === 'lowdefy.yml') {
+        content = await getConfigFile({
             context,
             refDef,
             referencedFrom
         });
-        return parseRefContent({
-            content,
-            refDef
+    } else if (refDef.resolver || context.refResolver) {
+        content = await runRefResolver({
+            context,
+            refDef,
+            referencedFrom
+        });
+    } else {
+        content = await getConfigFile({
+            context,
+            refDef,
+            referencedFrom
         });
     }
-    const cacheKey = getCacheKey(refDef);
-    const cached = context.parsedContentCache.get(cacheKey);
-    if (cached !== undefined) {
-        return cached;
-    }
-    const rawContent = await getConfigFile({
-        context,
-        refDef,
-        referencedFrom
-    });
-    const parsed = parseRefContent({
-        content: rawContent,
+    return parseRefContent({
+        content,
         refDef
     });
-    context.parsedContentCache.set(cacheKey, parsed);
-    return parsed;
 }
 export default getRefContent;

package/dist/build/buildRefs/getRefsFromFile.js

@@ -12,14 +12,16 @@
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-*/ import { type } from '@lowdefy/helpers';
+*/ import { serializer, type } from '@lowdefy/helpers';
 import makeRefDefinition from './makeRefDefinition.js';
 function getRefsFromFile(fileContent, parentRefDefId, refMap) {
     const foundRefs = [];
     const reviver = (key, value)=>{
         if (type.isObject(value)) {
             if (!type.isUndefined(value._ref)) {
-                const def = makeRefDefinition(value._ref, parentRefDefId, refMap);
+                // Capture line number from the object containing the _ref
+                const lineNumber = value['~l'];
+                const def = makeRefDefinition(value._ref, parentRefDefId, refMap, lineNumber);
                 foundRefs.push(def);
                 return {
                     _ref: def
@@ -28,7 +30,10 @@ function getRefsFromFile(fileContent, parentRefDefId, refMap) {
         }
         return value;
     };
-    const fileContentBuiltRefs = JSON.parse(JSON.stringify(fileContent), reviver);
+    // Use serializer.copy to preserve non-enumerable properties like ~l
+    const fileContentBuiltRefs = serializer.copy(fileContent, {
+        reviver
+    });
     return {
         foundRefs,
         fileContentBuiltRefs

package/dist/build/buildRefs/makeRefDefinition.js

@@ -14,10 +14,17 @@
   limitations under the License.
 */ import { get } from '@lowdefy/helpers';
 import getRefPath from './getRefPath.js';
-import makeRefHash from '../../utils/makeRefHash.js';
-function makeRefDefinition(refDefinition, parent, refMap) {
+import makeId from '../../utils/makeId.js';
+function makeRefDefinition(refDefinition, parent, refMap, lineNumber) {
+    const id = makeId();
     const refDef = {
         parent,
+        lineNumber
+    };
+    refMap[id] = refDef;
+    return {
+        ...refDef,
+        id,
         key: get(refDefinition, 'key'),
         original: refDefinition,
         path: getRefPath(refDefinition),
@@ -27,11 +34,5 @@ function makeRefDefinition(refDefinition, parent, refMap) {
             default: {}
         })
     };
-    const refHash = makeRefHash(refDef);
-    refMap[refHash] = {
-        parent
-    };
-    refDef.hash = refHash;
-    return refDef;
 }
 export default makeRefDefinition;

package/dist/build/buildRefs/parseRefContent.js

@@ -15,8 +15,67 @@
 */ /* eslint-disable no-param-reassign */ import { type } from '@lowdefy/helpers';
 import { getFileExtension, getFileSubExtension } from '@lowdefy/node-utils';
 import JSON5 from 'json5';
-import YAML from 'yaml';
+import YAML, { isMap, isSeq, isPair, isScalar } from 'yaml';
 import parseNunjucks from './parseNunjucks.js';
+function getLineNumber(content, offset) {
+    if (offset == null || offset < 0) return null;
+    return content.substring(0, offset).split('\n').length;
+}
+function addLineNumbers(node, content, result) {
+    if (isMap(node)) {
+        const obj = result || {};
+        if (node.range) {
+            Object.defineProperty(obj, '~l', {
+                value: getLineNumber(content, node.range[0]),
+                enumerable: false,
+                writable: true,
+                configurable: true
+            });
+        }
+        for (const pair of node.items){
+            if (isPair(pair) && isScalar(pair.key)) {
+                const key = pair.key.value;
+                const value = pair.value;
+                if (isMap(value)) {
+                    obj[key] = addLineNumbers(value, content, {});
+                } else if (isSeq(value)) {
+                    obj[key] = addLineNumbers(value, content, []);
+                } else if (isScalar(value)) {
+                    obj[key] = value.value;
+                } else {
+                    obj[key] = value?.toJSON?.() ?? value;
+                }
+            }
+        }
+        return obj;
+    }
+    if (isSeq(node)) {
+        const arr = result || [];
+        for (const item of node.items){
+            if (isMap(item)) {
+                arr.push(addLineNumbers(item, content, {}));
+            } else if (isSeq(item)) {
+                arr.push(addLineNumbers(item, content, []));
+            } else if (isScalar(item)) {
+                arr.push(item.value);
+            } else {
+                arr.push(item?.toJSON?.() ?? item);
+            }
+        }
+        return arr;
+    }
+    if (isScalar(node)) {
+        return node.value;
+    }
+    return node?.toJSON?.() ?? node;
+}
+function parseYamlWithLineNumbers(content) {
+    const doc = YAML.parseDocument(content);
+    if (doc.errors && doc.errors.length > 0) {
+        throw new Error(doc.errors[0].message);
+    }
+    return addLineNumbers(doc.contents, content);
+}
 function parseRefContent({ content, refDef }) {
     const { path, vars } = refDef;
     if (type.isString(path)) {
@@ -26,7 +85,7 @@ function parseRefContent({ content, refDef }) {
             ext = getFileSubExtension(path);
         }
         if (ext === 'yaml' || ext === 'yml') {
-            content = YAML.parse(content);
+            content = parseYamlWithLineNumbers(content);
         }
         if (ext === 'json') {
             content = JSON5.parse(content);

package/dist/build/buildRefs/populateRefs.js

@@ -12,32 +12,35 @@
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-*/ import { get, type } from '@lowdefy/helpers';
+*/ import { get, serializer, type } from '@lowdefy/helpers';
 function refReviver(key, value) {
     if (type.isObject(value)) {
         if (!type.isUndefined(value._ref)) {
-            return this.refCache.get(value._ref.hash);
+            return this.parsedFiles[value._ref.id];
         }
         if (value._var) {
             if (type.isString(value._var)) {
-                return JSON.parse(JSON.stringify(get(this.vars, value._var, {
+                return serializer.copy(get(this.vars, value._var, {
                     default: null
-                })));
+                }));
             }
             if (type.isObject(value._var) && type.isString(value._var.key)) {
-                return JSON.parse(JSON.stringify(get(this.vars, value._var.key, {
+                return serializer.copy(get(this.vars, value._var.key, {
                     default: type.isNone(value._var.default) ? null : value._var.default
-                })));
+                }));
             }
             throw new Error(`"_var" operator takes a string or object with "key" field as arguments. Received "${JSON.stringify(value)}"`);
         }
     }
     return value;
 }
-function populateRefs({ refCache, refDef, toPopulate }) {
-    return JSON.parse(JSON.stringify(toPopulate), refReviver.bind({
-        refCache,
-        vars: refDef.vars
-    }));
+function populateRefs({ parsedFiles, refDef, toPopulate }) {
+    // Use serializer.copy to preserve non-enumerable properties like ~r, ~k, ~l
+    return serializer.copy(toPopulate, {
+        reviver: refReviver.bind({
+            parsedFiles,
+            vars: refDef.vars
+        })
+    });
 }
 export default populateRefs;

package/dist/build/buildRefs/recursiveBuild.js

@@ -12,92 +12,93 @@
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-*/ import { type } from '@lowdefy/helpers';
+*/ import { serializer, type } from '@lowdefy/helpers';
 import evaluateBuildOperators from './evaluateBuildOperators.js';
 import getKey from './getKey.js';
 import getRefContent from './getRefContent.js';
 import getRefsFromFile from './getRefsFromFile.js';
 import populateRefs from './populateRefs.js';
 import runTransformer from './runTransformer.js';
-async function recursiveBuild({ context, refDef, count, referencedFrom, refCache, profiler }) {
-    const time = profiler?.time ?? ((_, fn)=>fn());
-    const timeSync = profiler?.timeSync ?? ((_, fn)=>fn());
-    // TODO: Maybe it would be better to detect a cycle, since this is the real issue here?
+async function recursiveBuild({ context, refDef, count, referencedFrom, refChainSet = new Set(), refChainList = [] }) {
+    // Detect circular references by tracking the chain of files being resolved
+    const currentPath = refDef.path;
+    if (refChainSet.has(currentPath)) {
+        const chainDisplay = [
+            ...refChainList,
+            currentPath
+        ].join(' -> ');
+        throw new Error(`Circular reference detected: ${chainDisplay}`);
+    }
+    refChainSet.add(currentPath);
+    refChainList.push(currentPath);
+    // Keep count as a fallback safety limit
     if (count > 10000) {
         throw new Error(`Maximum recursion depth of references exceeded.`);
     }
-    if (refCache.has(refDef.hash)) {
-        return refCache.get(refDef.hash);
-    }
-    let fileContent = await time('getRefContent', ()=>getRefContent({
-            context,
-            refDef,
-            referencedFrom
-        }));
-    const { foundRefs, fileContentBuiltRefs } = timeSync('getRefsFromFile', ()=>getRefsFromFile(fileContent, refDef.hash, context.refMap));
+    let fileContent = await getRefContent({
+        context,
+        refDef,
+        referencedFrom
+    });
+    const { foundRefs, fileContentBuiltRefs } = getRefsFromFile(fileContent, refDef.id, context.refMap);
+    const parsedFiles = {};
     // Since we can have references in the variables of a reference, we need to first parse
     // the deeper nodes, so we can use those parsed files in references higher in the tree.
     // To do this, since foundRefs is an array of ref definitions that are in order of the
     // deepest nodes first we for loop over over foundRefs one by one, awaiting each result.
     for (const newRefDef of foundRefs.values()){
         // Parse vars and path before passing down to parse new file
-        const parsedRefDef = timeSync('populateRefs:vars', ()=>populateRefs({
-                toPopulate: newRefDef,
-                refCache,
-                refDef
-            }));
-        context.refMap[parsedRefDef.hash].path = parsedRefDef.path;
-        // Build dependency graph: when refDef.path references parsedRefDef.path,
-        // add refDef.path as a dependent of parsedRefDef.path
-        if (parsedRefDef.path && refDef.path) {
-            if (!context.dependencyGraph.has(parsedRefDef.path)) {
-                context.dependencyGraph.set(parsedRefDef.path, new Set());
-            }
-            context.dependencyGraph.get(parsedRefDef.path).add(refDef.path);
-        }
+        const parsedRefDef = populateRefs({
+            toPopulate: newRefDef,
+            parsedFiles,
+            refDef
+        });
+        context.refMap[parsedRefDef.id].path = parsedRefDef.path;
         const parsedFile = await recursiveBuild({
             context,
             refDef: parsedRefDef,
             count: count + 1,
             referencedFrom: refDef.path,
-            refCache,
-            profiler
+            refChainSet,
+            refChainList
+        });
+        const transformedFile = await runTransformer({
+            context,
+            input: parsedFile,
+            refDef: parsedRefDef
         });
-        const transformedFile = await time('runTransformer', ()=>runTransformer({
-                context,
-                input: parsedFile,
-                refDef: parsedRefDef
-            }));
         // Evaluated in recursive loop for better error messages
-        const evaluatedOperators = await time('evaluateBuildOperators', ()=>evaluateBuildOperators({
-                context,
-                input: transformedFile,
-                refDef: parsedRefDef
-            }));
-        const withRefKey = timeSync('getKey', ()=>getKey({
-                input: evaluatedOperators,
-                refDef: parsedRefDef
-            }));
-        timeSync('cacheWithReviver', ()=>{
-            const reviver = (_, value)=>{
-                if (!type.isObject(value)) return value;
-                Object.defineProperty(value, '~r', {
-                    value: refDef.hash,
-                    enumerable: false,
-                    writable: true,
-                    configurable: true
-                });
-                return value;
-            };
-            refCache.set(newRefDef.hash, JSON.parse(JSON.stringify(withRefKey), reviver));
+        const evaluatedOperators = await evaluateBuildOperators({
+            context,
+            input: transformedFile,
+            refDef: parsedRefDef
+        });
+        const withRefKey = getKey({
+            input: evaluatedOperators,
+            refDef: parsedRefDef
+        });
+        const reviver = (_, value)=>{
+            if (!type.isObject(value)) return value;
+            Object.defineProperty(value, '~r', {
+                value: refDef.id,
+                enumerable: false,
+                writable: true,
+                configurable: true
+            });
+            return value;
+        };
+        // Use serializer.copy to preserve non-enumerable properties like ~l
+        parsedFiles[newRefDef.id] = serializer.copy(withRefKey, {
+            reviver
         });
     }
-    const result = timeSync('populateRefs:final', ()=>populateRefs({
-            toPopulate: fileContentBuiltRefs,
-            refCache,
-            refDef
-        }));
-    refCache.set(refDef.hash, result);
-    return result;
+    // Backtrack: remove current file from chain so sibling refs can use it
+    refChainSet.delete(currentPath);
+    refChainList.pop();
+    return populateRefs({
+        toPopulate: fileContentBuiltRefs,
+        parsedFiles,
+        refDef
+    });
 }
 export default recursiveBuild;

package/dist/build/buildTypes.js

@@ -14,18 +14,35 @@
   limitations under the License.
 */ import basicTypes from '@lowdefy/blocks-basic/types';
 import loaderTypes from '@lowdefy/blocks-loaders/types';
+import findSimilarString from '../utils/findSimilarString.js';
+import formatBuildError from './formatBuildError.js';
 function buildTypeClass(context, { counter, definitions, store, typeClass, warnIfMissing = false }) {
     const counts = counter.getCounts();
+    const definedTypes = Object.keys(definitions);
     Object.keys(counts).forEach((typeName)=>{
         if (!definitions[typeName]) {
+            let message = `${typeClass} type "${typeName}" was used but is not defined.`;
+            const suggestion = findSimilarString({
+                input: typeName,
+                candidates: definedTypes
+            });
+            if (suggestion) {
+                message += ` Did you mean "${suggestion}"?`;
+            }
+            const formattedError = formatBuildError({
+                context,
+                counter,
+                typeName,
+                message
+            });
             if (warnIfMissing) {
                 if (typeName === '_id') {
                     return;
                 }
-                context.logger.warn(`${typeClass} type "${typeName}" was used but is not defined.`);
+                context.logger.warn(formattedError);
                 return;
             }
-            throw new Error(`${typeClass} type "${typeName}" was used but is not defined.`);
+            throw new Error(formattedError);
         }
         store[typeName] = {
             originalTypeName: definitions[typeName].originalTypeName,

package/dist/build/formatBuildError.js

@@ -0,0 +1,34 @@
+/*
+  Copyright 2020-2024 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/ import { resolveConfigLocation } from '@lowdefy/helpers';
+function formatBuildError({ context, counter, typeName, message }) {
+    const configKey = counter.getLocation(typeName);
+    if (!configKey) {
+        return `[Config Error] ${message}`;
+    }
+    const location = resolveConfigLocation({
+        configKey,
+        keyMap: context.keyMap,
+        refMap: context.refMap,
+        configDirectory: context.directories.config
+    });
+    if (!location) {
+        return `[Config Error] ${message}`;
+    }
+    const source = location.source ? `${location.source} at ${location.config}` : '';
+    const link = location.link || '';
+    return `[Config Error] ${message}\n  ${source}\n  ${link}`;
+}
+export default formatBuildError;

package/dist/build/writeLogger.js

@@ -0,0 +1,19 @@
+/*
+  Copyright 2020-2024 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/ import { serializer } from '@lowdefy/helpers';
+async function writeLogger({ components, context }) {
+    await context.writeBuildArtifact('logger.json', serializer.serializeToString(components.logger ?? {}));
+}
+export default writeLogger;

package/dist/createContext.js

@@ -13,30 +13,17 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 */ import { mergeObjects } from '@lowdefy/helpers';
-import { BuildParser } from '@lowdefy/operators';
-import operators from '@lowdefy/operators-js/operators/build';
 import createCounter from './utils/createCounter.js';
 import createReadConfigFile from './utils/readConfigFile.js';
 import createWriteBuildArtifact from './utils/writeBuildArtifact.js';
 import defaultTypesMap from './defaultTypesMap.js';
-function createContext({ customTypesMap, directories, logger, refResolver, stage = 'prod', changedFiles = [], buildState = null }) {
-    const operatorsParser = new BuildParser({
-        env: process.env,
-        operators
-    });
-    // Use persistent buildState if provided (for incremental builds in dev mode)
-    // Otherwise create fresh Maps
-    const dependencyGraph = buildState?.dependencyGraph ?? new Map();
-    const parsedContentCache = buildState?.parsedContentCache ?? new Map();
+function createContext({ customTypesMap, directories, logger, refResolver, stage = 'prod' }) {
     const context = {
-        changedFiles,
-        dependencyGraph,
+        connectionIds: new Set(),
         directories,
         jsMap: {},
         keyMap: {},
         logger,
-        operatorsParser,
-        parsedContentCache,
         readConfigFile: createReadConfigFile({
             directories
         }),