@naanlang/naan 1.6.0 → 1.7.1

package/frameworks/project/build.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2019-2024 by Richard C. Zulch
+ * Copyright (c) 2019-2026 by Richard C. Zulch
  *
  */
 
@@ -304,18 +304,26 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
     //
     // Apply substitutions from readSubstitutionFile to string, returning a tuple of the resulting
     // text and the number of substitution patterns actually used from the supplied dictionary.
-    //
-    function applySubstitutions(text, subs, local subbed) {
-        subbed = { }
-        list(text.replace(RegExp("[$][^$\\s]+[$]", "g"), function(match, local subst) {
-            subst = subs[match.slice(1,-1)]
-            if subst {
-                subbed[match] = true
-                subst
-            }
-            else
-                match
-        }), length(subbed))
+    // Specify iterative to repeat the substitution on the result until no more substitutions occur.
+    //
+    function applySubstitutions(text, subs, iterative, local count, subbed) {
+        count = 0
+        loop {
+            subbed = { }
+            text = text.replace(RegExp("[$][^$\\s]+[$]", "g"), function(match, local subst) {
+                subst = subs[match.slice(1,-1)]
+                if subst {
+                    subbed[match] = true
+                    subst
+                }
+                else
+                    match
+            })
+            count += length(subbed)
+            if length(subbed) == 0 || !iterative
+                break
+        }
+        list(text, count)
     }
     
     //
@@ -426,7 +434,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
     // autoquoted to itself, a symbol. This can be used to detect build time vs. runtime.
     
     function parseAndPack1(text, fpath, options, operation,
-        local util, lib, penv, naansym, dlogsym, pmod, pcomp, output, textout, annerror, annotations, package)
+        local util, lib, penv, naansym, dlogsym, pmod, pcomp, ournn, output, textout, annerror, annotations, package)
     {
         //
         // textline
@@ -445,20 +453,39 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
         lib = Naan.module.modlist().Lib.exports
         if !util || !lib
             return (list("Build packager cannot access support functions"))
+        //
+        // The following crazy rigmarole is about customizing the ParsEnvironment so we can glean
+        // enough information to build a package without actually executing the code. To do this we
+        // replace the entire Naan.module collection of functions with just build, so the others do
+        // nothing. From that we collect the module and component names, plus the manifest. This 
+        // also sets up a benign "exports" dictionary so module build can "go through the mtions"
+        // without modification.
+        //
         penv = lib.ParsEnvironment(Dialect, "build-".concat(operation))
         penv.evalq(letlocal(App))                                           // make local copy of App
         penv.evalq(letlocal(js))                                            // make local copy of js
-        naansym = penv.evalq(letlocal(Naan).0)                              // make local copy of Naan
-        penv.evalq(assign(naansym,new(Naan)))                               // make it modifiable
+        ournn = new(Naan)                                                   // get modifiable Naan
         pmod = { }                                                          // our module object
         pcomp = { }                                                         // our component object
-        car(naansym).module = {
+        ournn.module = {
             build: function(modname, compname, proc) {
-                pmod.id = module
+                pmod.id = module.id
+                pmod.exports = { }
                 pcomp.name = compname
                 call(proc, pmod, pcomp)                                     // obtain manifest
             }
         }
+        naansym = penv.evalq(function(nn) {
+            assign(letlocal(Naan).0,nn)
+            `Naan
+        }(ournn))                                                           // make Naan modifiable
+        penv.evalq(assign(letlocal(module).0, {                             // make local copy of module
+            exports: { }
+        }))
+        //
+        // Here we make a custom debuglog function that identifies the text without color and will
+        // write the output to the build log.
+        //
         dlogsym = penv.evalq(letlocal(debuglog).0)                          // make local copy of debuglog
         penv.evalq(function \.debuglog args {                               // redefine debuglog for capture
             let (arg, spaced) {
@@ -471,8 +498,19 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
                 apply(textline, reverse(spaced))
             } () })
         penv.evalq(putproc(car(dlogsym), \.debuglog.proc))
+        //
+        // The NideBuild symbol is "debug" or "release", etc. or is simply autoquoted as the symbol
+        // itself when not building. This is used if the source needs different behavior at build
+        // time or run time.
+        //
         penv.eval(append(quote(assign(compress("NideBuild"))), operation))  // NideBuild symbol is operation name string
+        //
+        // This is the actual meat of the matter, the parsing of the program text.
+        //
         output = penv.parse(text)                                           // parse and evaluate
+        //
+        // Output processing and error logging
+        //
         textout = ""
         try {
             if !pcomp.name
@@ -507,20 +545,30 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
             }
             if output.errors.length > 0
                 return (list("errors in ".concat(fpath), false, annotations, textout))
+            //
+            // Build the package, if not just doing syntax checking
+            //
             if options.naanpack || options.jspack {                         // actually building a package?
                 if !pcomp.name {
-                    pcomp = penv.evalq(car(compress(NaanBuildComponent)))   // fallback for building Module.nlg itself
+                    //
+                    // Some components cannot use a build function because they are too early in the
+                    // boot process, but we still need a component. We look for NaanBuildComponent
+                    // to define a dictionary with filename, name and manifest fields. For example,
+                    // see Util:Module.nlg
+                    //
+                    pcomp = penv.evalq(car(compress(NaanBuildComponent)))
                     if !pcomp || !fpath.endsWith("/".concat(pcomp.filename))
                         return (list("Build cannot package without component in ".concat(fpath), false, annotations, textout))
                 }
-                package = penv.evalq(util.pkgSave(compress(pcomp.name), pcomp.manifest))
+                package = util.pkgSave(compress(pcomp.name), pcomp.manifest)
+                if string(package)
+                    return (list("Build cannot generate package for ".concat(fpath), false, annotations, textout))
             }
         } catch {
             if true
                 return (list("Packaging exception in ".concat(fpath, ": ", tostring(exception)), false, annotations, textout))
         } finally {
             penv.destroy()                                                  // remove the namespace
-            debuglog("parseAndPack1 done", fpath)
         }
         if options.naanpack
             package = (js.w||js.s||js.g).JSON.stringify(package)            // doesn't have frameworks/common available
@@ -546,7 +594,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
             build.parallel = ExecutorParallel(track, {                      // create worker pool
                 basename: "build"
                 initeval: `(false, (parseAndPack1))
-            }) 
+            })
         result = build.parallel.rpc(parseAndPack1, list(text, fpath, options, operation))
         if !result.0 {
             result = result.1                                               // unpeel RPC result
@@ -1004,9 +1052,9 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
                 }
                 else if ((`(error, content) = fs.readFile(entry.readpath, readOptions)) && !error) {
                     if entry.license
-                        `(content) = applySubstitutions(content, licFileDefs)
+                        `(content) = applySubstitutions(content, licFileDefs, true)
                     if entry.version
-                        `(content) = applySubstitutions(content, verFileDefs)
+                        `(content) = applySubstitutions(content, verFileDefs, true)
                     if entry.naanpack
                         `(error, content) = parseAndPack(content, entry.readpath, { naanpack: true })
                     else if entry.jspack
@@ -1211,17 +1259,20 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
     //
     // Run the current build product by spawning a new worker.
     //
-    build.runSpawn = function runSpawn(local error, maintext, executor, vpath, site) {
+    build.runSpawn = function runSpawn(stage,
+        local error, maintext, executor, vpath, site, dirpath) {
         `(error, maintext) = fs.readFile(JSpath.join(build.destpath, build.runway.main))
         if error
             error = Error("Builder: cannot read main file", error)
         else {
+            dirpath = fs.path.resolve("/", fs.rootpath, build.destpath, build.runway.path || "")
             `(error, executor) = track.spawn(build.runway.spawn.0, "${rules.name} ${build.version}", {
                 workerID: "Run-${build.version}+${build.buildnum}"
                 startup: {
-                    initcmds: maintext
-                    dirpath: fs.path.resolve("/", fs.rootpath, build.destpath)
+                    maintext: maintext
+                    dirpath: dirpath
                 }
+                rootPath: dirpath
             })
             if error
                 error = Error("Builder: cannot spawn executor instance", error)
@@ -1249,13 +1300,12 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
     //
     // Run the current build product by opening a new window.
     //
-    build.runWindow = function runWindow(
+    build.runWindow = function runWindow(stage,
         local title, main, url, features, vpath, error, window, site) {
         if !js.w
             return (list(Error('Run method "window" requires browser host')))
         if !build.runway."v-site"
             return (list(Error('Run method "window" requires v-site specification')))
-        title = rules.name.concat("-", build.buildnum)
         main = build.runway.main
         if !main
             main = "index.html"
@@ -1271,6 +1321,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
         if error
             return (list(Error("Builder: cannot create virtual site", error)))
         sleep(1)                                                            // needed on Windows or it won't open, fiik why
+        title = rules.name.concat("-", stage, "-", operation)
         window = js.w.open(url, title, features)
         list(false, { ok: true })                                           // we don't know the executor
     }
@@ -1281,13 +1332,13 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
     // Run the current build product, which would be after the make call. Returns a standard result
     // tuple.
     //
-    build.run = function run() {
+    build.run = function run(stage) {
         if build.runway.open
             fs.shellOpen(JSpath.join(build.destpath, build.runway.open), build.runway.args)
         else if build.runway.spawn
-            build.runSpawn()
+            build.runSpawn(stage)
         else if build.runway.window
-            build.runWindow()
+            build.runWindow(stage)
         else
             list(false, { ok: true })
     }

package/frameworks/project/projects.nlg

@@ -296,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.6.0+1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.7.1+1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info
@@ -646,7 +646,7 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
             if builder.refresh()
                 buildonly = !buildonly                                      // reverse if refresh
             if !buildonly
-                `(error) = builder.run()
+                `(error) = builder.run(stage)
         }
         builder.destroy()
         procon.builder = false

package/frameworks/running/debugcom.nlg

@@ -19,9 +19,6 @@
 
 closure DebugCommand(executor, notify, local decom, excon) {
     decom = new(object, this)
-    executor.DebugCmd({
-        op: "attach"
-    })
     decom.executor = executor
 
     // debugData
@@ -96,6 +93,24 @@ closure DebugCommand(executor, notify, local decom, excon) {
         status
     }
 
+    // query
+    // Send a status update back.
+    //
+    decom.query = function query() {
+        decom.executor.DebugCmd({
+            op: "query"
+        })
+    }
+    
+    // attach
+    // Attach our GUI to the low-level debugger
+    //
+    decom.attach = function attach() {
+        executor.DebugCmd({
+            op: "attach"
+        })
+    }
+
     //
     // Run control.
     //
@@ -138,6 +153,8 @@ closure DebugCommand(executor, notify, local decom, excon) {
         notify.runState(debugStatus())
     }
 
+    attach()
+
     // finis
     
     decom

package/frameworks/running/debugnub.nlg

@@ -35,6 +35,7 @@ closure DebugNub(local nub, debug, paused) {
     nub.watches = []
     nub.stacks = { }
     nub.usage = { }
+    paused = { }
 
     // debugDepth
     // Return the recursive debugging depth as an integer.
@@ -45,6 +46,18 @@ closure DebugNub(local nub, debug, paused) {
             curdepth = 0
         curdepth
     }
+    
+    // sendStatus
+    // Send current status to the far end.
+    //
+    function sendStatus() {
+        js.t.ReplyDebugger({
+            op: "status"
+            depth: debugDepth()
+            bpenable: debug.bpenable
+            exenable: nub.exenable
+        })
+    }
 
     // onDebugger proc
     // Process an incoming debugger command.
@@ -52,17 +65,13 @@ closure DebugNub(local nub, debug, paused) {
     function onDebugger(msg) {
         if msg.op == "attach" {
             debug.setCommander(nub)
-            js.t.ReplyDebugger({
-                op: "status"
-                depth: debugDepth()
-                bpenable: debug.bpenable
-                exenable: nub.exenable
-            })
+            sendStatus()
         }
         else if msg.op == "detach" {
             debug.setCommander(false)
             paused.wait = false                                             // false: go back to ndb
-        }
+        } else if msg.op == "query"
+            sendStatus()
         else if msg.op == "pause"
             debugger()
         else if msg.op == "resume" || msg.op == "stop"

package/frameworks/running/executors.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2024 by Richard C. Zulch
+ * Copyright (c) 2020-2026 by Richard C. Zulch
  *
  */
 
@@ -23,13 +23,21 @@ closure ExecutorBase(track, type, name, local exec) {
     exec.type = type
     exec.name = name
     exec.contexts = { }
-    
+
     // execReset
     //
     // Initialize remote communications for a new or restarted instance.
     //
-    exec.execReset = function execReset() {
+    exec.execReset = function execReset(error, local xid, pending) {
+        if !error
+            error = Error("execReset")
+        for `(xid, pending) in exec.pending {
+            pending.signal(list(error))
+            exec.pending[xid] = undefined
+        }
         exec.pending = { }
+        exec.inst_ready = false
+        exec.inst_error = undefined
         exec.nextSendID = toint(RandomBytes(3))                             // keep our sequence code unique for shared servers
     }
     execReset()
@@ -52,7 +60,7 @@ closure ExecutorBase(track, type, name, local exec) {
             pending.signal(Error("execSend failed", error))
         pending
     }
-    
+
     // execReceived
     //
     // The specified message was received from the instance, so signal it.
@@ -65,7 +73,7 @@ closure ExecutorBase(track, type, name, local exec) {
                 exec.reverseDispatch = require("./taskexec.nlg").TaskDispatcher(function(data) {
                     data.reverse = true
                     exec.PostMessage(data)
-                }, exec.contexts[data.contextID].coder)
+                }, exec.contexts[data.contextID].coder, "execReceived")
             data.reverse = false
             exec.reverseDispatch(data)
             return
@@ -73,7 +81,7 @@ closure ExecutorBase(track, type, name, local exec) {
         pending = exec.pending[data.xid]
         if !pending {                                                       // not waiting on message
             if data.xmsg == "xready"
-                exec.pending.ready = true
+                exec.inst_ready = true
             if data.xmsg == "xattention"
                 { }
             return
@@ -81,7 +89,7 @@ closure ExecutorBase(track, type, name, local exec) {
         exec.pending[data.xid] = undefined
         pending.signal(list(false, data))
     }
-    
+
     // execSetStatus
     //
     // Received a status update from the instance. This makes a copy of the specified dictionary for
@@ -93,29 +101,25 @@ closure ExecutorBase(track, type, name, local exec) {
     exec.execSetStatus = function execSetStatus(status) {
         exec.xstatus = new(status)                                          // for GC (!)
     }
-    
+
     // execFailed
     //
     // Communication has failed.
     //
     exec.execFailed = closure execFailed(error, local xid, pending) {
-        exec.pending.error = error
+        exec.inst_error = error
         for `(xid, pending) in exec.pending
             pending.signal(list(error))
     }
-    
+
     // execClosed
     //
-    //
     // Note that the connection has closed, so signal all the waiters.
     //
-    exec.execClosed = closure execClosed(error, local xid, pending) {
-        for `(xid, pending) in exec.pending {
-            pending.signal(list(error))
-            pending[xid] = undefined
-        }
+    exec.execClosed = closure execClosed(error) {
+        execReset(error)
     }
-        
+
     // context
     //
     // Return a context object for the instance, or an error, in a result tuple. The optional state
@@ -127,7 +131,7 @@ closure ExecutorBase(track, type, name, local exec) {
         retries = 0
         ms = milliseconds()
         delta = 1
-        while !exec.pending.ready {
+        while !exec.inst_ready {
             `(error) = exec.PostMessage({
                 xmsg: "xstart"
             })
@@ -135,8 +139,8 @@ closure ExecutorBase(track, type, name, local exec) {
                 return (Error("exec.context failed", error))
             if ++retries > 100
                 return (list(Error("timeout creating remote context", type, name)))
-            if exec.pending.error
-                return (list(Error("cannot create remote context", exec.pending.error)))
+            if exec.inst_error
+                return (list(Error("cannot create remote context", exec.inst_error)))
             sleep(delta)
             if delta < 100
                 delta *= 2 }
@@ -157,7 +161,7 @@ closure ExecutorBase(track, type, name, local exec) {
     exec.attention = function attention() {
         track.attention(exec)
     }
-    
+
     // finis
     exec
 };
@@ -220,6 +224,8 @@ closure ExecutorContext(exec, state, coder, local context, util) {
     // an optional tuple of symbols to retrieve after evaluation.
     //
     context.eval = closure evalf(expr, put, get, local roots, pkg, data, pending, result) {
+        if !exec.inst_ready
+            return (list(Error("execution context closed")))
         if tuple(put)
             roots = cons(`expr, put)
         else
@@ -288,6 +294,8 @@ closure ExecutorContext(exec, state, coder, local context, util) {
     // The procedure must be specified as a symbol or string.
     //
     context.rpc = closure rpc(proc, args, local data, pending, result) {
+        if !exec.inst_ready
+            return (list(Error("execution context closed")))
         data = {
             contextID: state.contextID
             proc: proc
@@ -566,13 +574,13 @@ closure ExecutorParallel(track, options, local maxcount, parallel, state, exqueu
             return (false)                                                  // already destroyed
         crValue = cancelResult
         maxcount = 0
-        state.available = false
-        parallel.contexts = false
         while state.workers.length > 0 {
             worker = state.workers.pop()
             parallel.contexts[worker] = false
             worker.destroy()
         }
+        state.available = false
+        parallel.contexts = false
         while exqueue.length > 0 {
             next = exqueue.pop()
             next.pending.signal(crValue)
@@ -604,12 +612,14 @@ closure ExecutorTracker(local xtrak) {
     // Notify the watchers by calling a named procedure
     
     closure notify(changes, local watch) {
-        for watch in xtrak.watchers
-            call(watch, changes)
+        future(function(){
+            for watch in xtrak.watchers
+                call(watch, changes)
+        }, 10)
     }
 
     // add
-    // Add an execution instance at the specified location.
+    // Add an execution instance.
 
     xtrak.add = closure add(executor, local instance) {
         instance = new(nonce)

package/frameworks/running/instances.nlg

@@ -41,6 +41,7 @@
 closure InTop(track, options, local intop, nlregex, str, error) {
     global(App)
     intop = ExecutorBase(track, options.type, options.name)
+    intop.verbose = false
     intop.workerID = options.workerID || undefined
     intop.msgQueue = []
     nlregex = RegExp("\\n", "g")
@@ -101,7 +102,7 @@ closure InTop(track, options, local intop, nlregex, str, error) {
     // Received an unknown destination notice from the other side.
     //
     intop.impUnknown = function impUnknown(unknownID, sourceID, gateway) {
-        if !gateway
+        if !gateway && intop.verbose
             debuglog("InTop.impUnknownID:", unknownID, "sourceID:", sourceID)
     }
 
@@ -263,8 +264,12 @@ closure InTop(track, options, local intop, nlregex, str, error) {
     //     A terminal was attached.
     //
     intop.termAttach = function termAttach(repl, db, notify, local msg) {
+        if !repl {
+            debuglog("termAttach false repl")
+            return (false)
+        }
         if !intop.msgQueue
-            return (list(Error("already attached")))
+            return (intop)                                                  // already attached
         intop.sender(intop.botDestID, {
             id: "attached",
             guid: intop.instanceID
@@ -294,7 +299,7 @@ closure InTop(track, options, local intop, nlregex, str, error) {
     //
     intop.termDetach = function termDetach() {
         if intop.msgQueue
-            return (list(Error("already detached")))
+            return                                                          // already detached
         intop.term.SetNotify()
         intop.msgQueue = []
         intop.debugger.detach(intop)
@@ -310,19 +315,17 @@ closure InTop(track, options, local intop, nlregex, str, error) {
 
     // close
     //
-    //     Close our access to the worker.
+    //     Close our access to the worker, but it can be reopened.
     //
     intop.close = function close() {
-        if !intop.instanceID
+        if !intop.botDestID
             return
         intop.sender(intop.botDestID, {
             id: "unsubscribe"
         })
-        intop.execClosed(Error("instance closed"))
-        track.delete(intop)
-        termDetach()
+        intop.botDestID = false
         App.imp.unregister(intop.instanceID)
-        intop.instanceID = false
+        intop.sender = false
     }
 
     // destroy
@@ -333,11 +336,17 @@ closure InTop(track, options, local intop, nlregex, str, error) {
         if intop.destroyed
             return
         intop.destroyed = true
+        intop.execClosed(Error("instance closed"))
         intop.sender(intop.botDestID, {
             id: "destroy",
             guid: intop.instanceID
         })
-        close()
+        track.delete(intop)
+        termDetach()
+        sleep(10)
+        App.imp.unregister(intop.instanceID)
+        intop.sender = false
+        intop.cleanup()                                                     // child classes release resources
     }
 
     // open
@@ -347,24 +356,27 @@ closure InTop(track, options, local intop, nlregex, str, error) {
     intop.open = closure open(inbotID) {
         if intop.destroyed
             return (list(Error("InTop already destroyed")))
-        if intop.instanceID {
-            if inbotID != intop.botDestID
-                return (list(Error("InTop already open")))
-        } else
-            intop.instanceID = UUID()
+        if intop.botDestID == inbotID
+            return (list(false, { reopen: true }))
+        if intop.botDestID
+            close()
         intop.botDestID = inbotID
-        intop.sender = App.imp.register(intop, {
-            name: options.name
-            type: options.type
-            instanceID: intop.instanceID
-            services: [ "naan-executor" ]
-        })
+        if !intop.instanceID {
+            intop.instanceID = App.imp.getUUID("InTop(${options.name}) instanceID")
+            track.add(intop)
+        }
+        if !intop.sender
+            intop.sender = App.imp.register(intop, {
+                name: options.name
+                type: options.type
+                instanceID: intop.instanceID
+                services: [ "naan-executor" ]
+            })
         `(error) = intop.sender(intop.botDestID, {
             id: "subscribe"
         })
         if error
             return(list(error))
-        track.add(intop)
         list(false, { ok: true })
     }