@naanlang/naan 1.0.0 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/LICENSE.md +1 -1
  2. package/README.md +37 -8
  3. package/bin/index.js +148 -0
  4. package/dist/env_web.js +145 -16
  5. package/dist/naan.min.js +5 -5
  6. package/frameworks/browser/sworker.js +309 -91
  7. package/frameworks/browser/terminals.nlg +22 -9
  8. package/frameworks/browser/workers.nlg +19 -11
  9. package/frameworks/client/apiclient.nlg +17 -4
  10. package/frameworks/client/psm_client.nlg +116 -53
  11. package/frameworks/common/common.nlg +12 -2
  12. package/frameworks/node/apiserver.nlg +25 -13
  13. package/frameworks/node/filesystem.nlg +173 -30
  14. package/frameworks/node/psm_server.nlg +95 -32
  15. package/frameworks/project/build.nlg +40 -23
  16. package/frameworks/project/projects.nlg +47 -26
  17. package/frameworks/running/debugnub.nlg +2 -5
  18. package/frameworks/storage/csv.nlg +120 -0
  19. package/frameworks/storage/dbt_pouch.nlg +41 -25
  20. package/frameworks/storage/psm.nlg +108 -28
  21. package/frameworks/storage/psm_dbtables.nlg +7 -3
  22. package/frameworks/storage/resources.nlg +30 -2
  23. package/lib/browser/env_web.js +147 -18
  24. package/lib/browser/env_webworker.js +1 -1
  25. package/lib/browser/require.js +1 -1
  26. package/lib/core/naanlib.js +5 -5
  27. package/lib/env_node.js +14 -2
  28. package/lib/node_repl.js +2 -2
  29. package/package.json +4 -1
  30. package/plugins/serviceAws/aws_cloudwatchlogs.nlg +1 -1
  31. package/plugins/serviceAws/aws_dynamo.nlg +625 -141
  32. package/plugins/serviceAws/aws_dynextra.nlg +294 -0
  33. package/plugins/serviceAws/dbt_aws.nlg +31 -11
  34. package/plugins/serviceAws/psm_aws.nlg +42 -26
  35. package/plugins/serviceAws/serviceAws.nlg +1 -0
  36. package/plugins/serviceGitHub/psm_github.nlg +41 -25
  37. package/plugins/serviceGitLab/psm_gitlab.nlg +41 -25
@@ -6,7 +6,7 @@
6
6
  *
7
7
  * column positioning: // // !
8
8
  *
9
- * Copyright (c) 2017-2021 by Richard C. Zulch
9
+ * Copyright (c) 2017-2022 by Richard C. Zulch
10
10
  *
11
11
  */
12
12
 
@@ -273,11 +273,48 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
273
273
  callback(false, { md5: md5data })
274
274
  })
275
275
  }
276
-
276
+
277
+ // readlink
278
+ //
279
+ // Read the contents of a symbolic link.
280
+
281
+ files.readlink = closure readlink(path, callback, local error, localpath) {
282
+ if !callback
283
+ return (syncAdapter(readlink, path))
284
+ `(error, localpath) = localPath(path)
285
+ if error
286
+ return (asyncResult(callback, error))
287
+ fs.readlink(localpath, function(err, linkString) {
288
+ if err
289
+ callback(Error("fs.readlink failed:", localpath, err))
290
+ else
291
+ callback(false, linkString)
292
+ })
293
+ }
294
+
295
+ // symlink
296
+ //
297
+ // Create a symbolic link "path" that points to "target". No options are currently supported.
298
+
299
+ files.symlink = closure symlink(target, path, options, callback,
300
+ local error, localpath) {
301
+ if !callback
302
+ return (syncAdapter(symlink, target, path, options))
303
+ `(error, localpath) = localPath(path)
304
+ if error
305
+ return (asyncResult(callback, error))
306
+ fs.symlink(target, localpath, function(err) {
307
+ if err
308
+ callback(Error("fs.symlink failed:", target, localpath, err))
309
+ else
310
+ callback(false, { ok: true })
311
+ })
312
+ }
313
+
277
314
  // delete
278
315
  //
279
316
  // Delete the file at the specified path, or a directory if the options dictionary includes:
280
- // recursive: true -- "recursively" delete directory and children
317
+ // recursive: true -- recursively delete directory and children
281
318
 
282
319
  files.delete = closure delete(path, options, callback, local error, localpath) {
283
320
  if !callback
@@ -286,9 +323,9 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
286
323
  if error
287
324
  return (asyncResult(callback, error))
288
325
  if options.recursive
289
- fs.rmdir(localpath, function(err) {
326
+ fs.rm(localpath, { recursive: true, force: true }, function(err) {
290
327
  if err
291
- callback(Error("fs.rmdir failed:", localpath, err))
328
+ callback(Error("fs.rm failed:", localpath, err))
292
329
  else
293
330
  callback(false, { ok: true })
294
331
  })
@@ -303,15 +340,22 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
303
340
 
304
341
  // info
305
342
  //
306
- // Report info on a file or directory.
343
+ // Report info on a file or directory. The options are:
344
+ // {
345
+ // follow: <boolean> // follow links
346
+ // }
307
347
 
308
- files.info = closure info(path, callback, local error, localpath) {
348
+ files.info = closure info(path, options, callback, local error, localpath, fsop) {
309
349
  if !callback
310
- return (syncAdapter(info, path))
350
+ return (syncAdapter(info, path, options))
311
351
  `(error, localpath) = localPath(path)
312
352
  if error
313
353
  return (asyncResult(callback, error))
314
- fs.lstat(localpath, function(err, stat, local data, info, nodetype) {
354
+ if options.follow
355
+ fsop = `stat
356
+ else
357
+ fsop = `lstat
358
+ fs[fsop](localpath, function(err, stat, local data, info, nodetype) {
315
359
  if err
316
360
  return (callback(Error("fs.stat failed:", localpath, err)))
317
361
  info = {
@@ -336,9 +380,102 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
336
380
  })
337
381
  }
338
382
 
383
+ // setInfo
384
+ //
385
+ // Set selected info on a file or directory. The info dictionary has the same layout as that
386
+ // returned by files.info(), and every applicable key defined in the dictionary is set. To omit
387
+ // setting some of the information, remove its key from the info dictionary. The applicable keys
388
+ // are:
389
+ // {
390
+ // mtimeMs: <milliseconds>
391
+ // mtime: <date>
392
+ // nodejs_stat: {
393
+ // mode: <number>
394
+ // uid: <uid>
395
+ // gid: <gid>
396
+ // atimeMs: <milliseconds>
397
+ // atime: <date>
398
+ // mtimeMs: <milliseconds>
399
+ // mtime: <date>
400
+ // }
401
+ // }
402
+ // If more than one timestamp exists of a certain type, the millisecond version is used. Some
403
+ // values are set together by the OS, like uid & guid, or mtime & atime. Specifying just one of
404
+ // these will cause setInfo to use the same value for both.
405
+
406
+ files.setInfo = closure setInfo(path, info, callback,
407
+ local error, localpath, mtime, atime, uid, gid) {
408
+ if !callback
409
+ return (syncAdapter(setInfo, path, info))
410
+ if !dictionary(info) && !xobject(info)
411
+ error = Error("setInfo: invalid argument:", typeof(info))
412
+ else
413
+ `(error, localpath) = localPath(path)
414
+ if error
415
+ return (asyncResult(callback, error))
416
+ atime = info.nodejs_stat.atimeMs/1000.0 || info.nodejs_stat.atime
417
+ mtime = info.mtimeMs/1000.0 || info.mtime
418
+ if !mtime
419
+ mtime = info.nodejs_stat.mtimeMs/1000.0 || info.nodejs_stat.mtime
420
+ if atime || mtime {
421
+ atime = atime || mtime
422
+ mtime = mtime || atime
423
+ `(error) = setTimes(path, atime, mtime)
424
+ }
425
+ if !error && info.nodejs_stat.mode
426
+ `(error) = setMode(path, info.nodejs_stat.mode)
427
+ uid = info.nodejs_stat.uid
428
+ gid = info.nodejs_stat.gid
429
+ if !error && (uid || gid) {
430
+ uid = uid || gid
431
+ gid = gid || uid
432
+ `(error) = setOwnerGroup(path, uid, gid)
433
+ }
434
+ if error
435
+ callback(Error("setInfo failed:", localpath, error))
436
+ else
437
+ callback(false, { ok: true})
438
+ }
439
+
440
+ // setOwnerGroup
441
+ //
442
+ // Set owner/group on a file or directory.
443
+
444
+ files.setOwnerGroup = closure setOwnerGroup(path, uid, gid, callback, local error, localpath) {
445
+ if !callback
446
+ return (syncAdapter(setOwnerGroup, path, uid, gid))
447
+ `(error, localpath) = localPath(path)
448
+ if error
449
+ return (asyncResult(callback, error))
450
+ fs.chown(localpath, uid, gid, function(err, stat) {
451
+ if err
452
+ callback(Error("fs.chown failed:", localpath, err))
453
+ else
454
+ callback(false, { ok: true})
455
+ })
456
+ }
457
+
458
+ // setMode
459
+ //
460
+ // Set mode bits on a file or directory.
461
+
462
+ files.setMode = closure setMode(path, mode, callback, local error, localpath) {
463
+ if !callback
464
+ return (syncAdapter(setMode, path, mode))
465
+ `(error, localpath) = localPath(path)
466
+ if error
467
+ return (asyncResult(callback, error))
468
+ fs.chmod(localpath, mode, function(err, stat) {
469
+ if err
470
+ callback(Error("fs.chmod failed:", localpath, err))
471
+ else
472
+ callback(false, { ok: true})
473
+ })
474
+ }
475
+
339
476
  // setTimes
340
477
  //
341
- // Set timestamps on a file or directory.
478
+ // Set timestamps on a file or directory. The timestamps can be dates or numeric seconds.
342
479
 
343
480
  files.setTimes = closure setTimes(path, atime, mtime, callback, local error, localpath) {
344
481
  if !callback
@@ -346,9 +483,6 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
346
483
  `(error, localpath) = localPath(path)
347
484
  if error
348
485
  return (asyncResult(callback, error))
349
- if !numeric(atime, mtime)
350
- return (asyncResult(callback, Error("invalid arguments:", atime, mtime)))
351
- localpath = JSpath.resolve(rootpath, localpath)
352
486
  fs.utimes(localpath, atime, mtime, function(err, stat) {
353
487
  if err
354
488
  callback(Error("fs.utimes failed:", localpath, err))
@@ -375,7 +509,7 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
375
509
  if name { // this record is fetchable
376
510
  ++pending.active
377
511
  filepath = JSpath.join(parentpath, name)
378
- files.info(filepath, function(error, info) {
512
+ files.info(filepath, false, function(error, info) {
379
513
  if error
380
514
  input[kdex].error = error // this record failed
381
515
  else
@@ -398,7 +532,7 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
398
532
 
399
533
  // dirList
400
534
  //
401
- // Return array of files in a directory if it exists. If options.head is true then this reads
535
+ // Return array of files in a directory if it exists. If options.stat is true then this reads
402
536
  // info for each file as well.
403
537
 
404
538
  files.dirList = closure dirList(path, options, callback, local error, localpath, output, nodelist, file) {
@@ -426,7 +560,7 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
426
560
  output.children.push({
427
561
  name: file
428
562
  })
429
- if options.head
563
+ if options.stat
430
564
  statfetch(output.children, localpath) // read all the info records
431
565
  callback(false, output)
432
566
  })
@@ -638,7 +772,7 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
638
772
  if dest.slice(-1) == JSpath.sep // new location, else entirely new pathname
639
773
  localdest = JSpath.join(localdest, JSpath.basename(srcpath))
640
774
  }
641
- fs.lstat(localdest, closure(err, stat, local cmd, args) {
775
+ fs.stat(localdest, closure(err, stat, local cmd, args) {
642
776
  if !err && !options.overwrite
643
777
  callback(Error("copy destination already exists",
644
778
  {
@@ -649,6 +783,7 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
649
783
  existing: localdest
650
784
  }))
651
785
  else {
786
+ debuglog("==> Warning: files.copy needs to be rewritten with fs primitives")
652
787
  if options.move {
653
788
  cmd = "mv"
654
789
  args = [localsrc, localdest] }
@@ -717,6 +852,7 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
717
852
  files.minimize = closure minimize(input, callback, local child, stdinStream) {
718
853
  if !callback
719
854
  return (syncAdapter(minimize, input))
855
+ debuglog("==> Warning: files.minimize is deprecated")
720
856
  child = nodecp.execFile("uglifyjs",
721
857
  ["--compress", "drop_debugger=false", "--mangle", "eval"], // ### uglifyjs options should be parameterized
722
858
  function(err, stdout, stderr) {
@@ -742,6 +878,7 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
742
878
  files.unzip = closure unzip(inpath, outpath, callback) {
743
879
  if !callback
744
880
  return (syncAdapter(unzip, inpath, outpath))
881
+ debuglog("==> Warning: files.unzip is deprecated")
745
882
  nodecp.execFile("unzip",
746
883
  ["-qo", inpath, "-d", outpath], // quiet; no queries
747
884
  { }, // execution options
@@ -757,21 +894,27 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
757
894
  //
758
895
  // touch
759
896
  //
760
- // Execute the touch utility, returning the result as a standard tuple: `(error, result)
897
+ // Touch the specified file or directory, returning the result as a standard result tuple
898
+ // `(error, result). When only the path is specified this sets the last-access and last-modified
899
+ // times to the current Date(). Otherwise the dates are set to the specified value(s). Note that
900
+ // this sets the times of a symbolic link file itself, not the file the link points to.
761
901
  //
762
- files.touch = closure touch(outpath, callback) {
902
+ files.touch = closure touch(path, atime, mtime, callback, local localpath) {
763
903
  if !callback
764
- return (syncAdapter(touch, outpath))
765
- nodecp.execFile("touch",
766
- [outpath],
767
- { }, // execution options
768
- function(err, stdout, stderr) {
769
- if err
770
- callback(Error("touch failed", err, stderr))
771
- else
772
- callback(false, { ok: true })
773
- }
774
- )
904
+ return (syncAdapter(touch, path, atime, mtime))
905
+ `(error, localpath) = localPath(path)
906
+ if error
907
+ return (asyncResult(callback, error))
908
+ if !atime && !mtime
909
+ atime = mtime = Date()
910
+ else if atime && !mtime
911
+ mtime = atime
912
+ fs.lutimes(localpath, atime, mtime, function(err) {
913
+ if err
914
+ callback(Error("touch failed", err))
915
+ else
916
+ callback(false, { ok: true })
917
+ })
775
918
  }
776
919
 
777
920
  //
@@ -6,7 +6,7 @@
6
6
  *
7
7
  * column positioning: // // !
8
8
  *
9
- * Copyright (c) 2019-2021 by Richard C. Zulch
9
+ * Copyright (c) 2019-2022 by Richard C. Zulch
10
10
  *
11
11
  */
12
12
 
@@ -32,7 +32,12 @@ closure psmsFsView(rootpath, local view, gitter) {
32
32
  // -- view.readLines(path, options, callback)
33
33
  // -- view.readFile(path, options, callback)
34
34
  // -- view.writeFile(path, data, options, callback)
35
- // -- view.info(path, callback)
35
+ // -- view.readlink(path, options, callback)
36
+ // -- view.symlink(target, path, options, callback)
37
+ // -- view.info(path, options, callback)
38
+ // -- view.setInfo(path, info, callback)
39
+ // -- view.setOwnerGroup(path, uid, gid, callback)
40
+ // -- view.setMode(path, mode, callback)
36
41
  // -- view.setTimes(path, atime, mtime, callback)
37
42
  // -- view.dirList(path, options, callback)
38
43
  // -- view.mkdir(path, callback)
@@ -118,7 +123,18 @@ closure psmsFsView(rootpath, local view, gitter) {
118
123
  options.encoding = req.query.encoding
119
124
  res.send(new(array, view.writeFile(path, req.body, options)))
120
125
  }
121
-
126
+
127
+ // op: readlink
128
+ function readlinkApi() {
129
+ res.send(new(array, view.readlink(path)))
130
+ }
131
+
132
+ // op: symlink
133
+ function symlinkApi(local options) {
134
+ options = { }
135
+ res.send(new(array, view.symlink(req.query.target, path, options)))
136
+ }
137
+
122
138
  // op: delete
123
139
  function deleteApi(local options) {
124
140
  options = { }
@@ -130,8 +146,26 @@ closure psmsFsView(rootpath, local view, gitter) {
130
146
  }
131
147
 
132
148
  // op: info
133
- function infoApi() {
134
- res.send(new(array, view.info(path)))
149
+ function infoApi(local options) {
150
+ options - { }
151
+ if req.query.follow
152
+ options.follow = req.query.follow
153
+ res.send(new(array, view.info(path, options)))
154
+ }
155
+
156
+ // op: setInfo
157
+ function setInfoApi() {
158
+ res.send(new(array, view.setInfo(path, req.body)))
159
+ }
160
+
161
+ // op: setOwnerGroup
162
+ function setOwnerGroupApi() {
163
+ res.send(new(array, view.setOwnerGroup(path, req.query.uid, req.query.gid)))
164
+ }
165
+
166
+ // op: setMode
167
+ function setModeApi() {
168
+ res.send(new(array, view.setMode(path, req.query.mode)))
135
169
  }
136
170
 
137
171
  // op: setTimes
@@ -141,8 +175,8 @@ closure psmsFsView(rootpath, local view, gitter) {
141
175
 
142
176
  // op: dirList
143
177
  function dirListApi(local options) {
144
- if req.query.head
145
- options = { head: true }
178
+ if req.query.stat
179
+ options = { stat: true }
146
180
  res.send(new(array, view.dirList(path, options)))
147
181
  }
148
182
 
@@ -219,8 +253,13 @@ closure psmsFsView(rootpath, local view, gitter) {
219
253
  readFile: readFileApi
220
254
  readLines: readLinesApi
221
255
  writeFile: writeFileApi
256
+ readlink: readlinkApi
257
+ symlink: symlinkApi
222
258
  delete: deleteApi
223
259
  info: infoApi
260
+ setInfo: setInfoApi
261
+ setOwnerGroup: setOwnerGroupApi
262
+ setMode: setModeApi
224
263
  setTimes: setTimesApi
225
264
  dirList: dirListApi
226
265
  mkdir: mkdirApi
@@ -263,38 +302,38 @@ closure psmsFsConnector(psm, api, local connClassID, connector, watch) {
263
302
  connector.vault = psm.vault(connClassID)
264
303
  connector.label = "Naan IDE Server"
265
304
  watch = psm.watchable(connector)
266
-
267
- // hashResourceID
268
- //
269
- // Make a stable hash based on the resource credentials that we can persist or share between
270
- // different domains accessing the same resource.
271
- connector.hashResourceID = function hashResourceID(resource, local ident) {
272
- ident = resource.urlName
273
- if resource.label
274
- ident = ident.concat(resource.label)
275
- HashSHA256(ident)
276
- }
277
-
305
+
278
306
  // findResource
279
307
  //
280
- // Find a resource with the specified contents, returning the resID or false. The loose criteria
281
- // is: if added, would the specified contents be redundant to an existing resource?
308
+ // Find a resource with the specified contents, returning the resID or false.
282
309
  connector.findResource = function findResource(resource, local resID, creds) {
283
310
  for resID in listResources().1 {
284
311
  creds = access(resID).1
285
- if resource.urlName == creds.urlName
286
- && (!resource.label || resource.label == creds.label)
312
+ if ((!resource.urlName || resource.urlName == creds.urlName)
313
+ && (!resource.label || resource.label == creds.label))
287
314
  return (resID)
288
315
  }
289
316
  false
290
317
  }
291
318
 
292
- // addResource
319
+ // writeResource
293
320
  //
294
- // Add credentials for a named resource. This is just a stub at the moment
295
-
296
- connector.addResource = function addResource(resource, local creds, errors, error, resID, data) {
321
+ // Add or update credentials for a named resource. If no resID is specified this adds a new
322
+ // resource; otherwise it updates an existing one. The resource is a dictionary comprising these
323
+ // keys:
324
+ // label - [optional] label we use for this resource
325
+ // urlName - URL (e.g. "http://host.com/node/fsapi")
326
+ // authSecret - access-control token for this URL
327
+ // hidden - [optional] don't enumerate to user
328
+ // locked - [optional] don't allow user delete
329
+ //
330
+ // The return value is an (error, resID) tuple. Error, if non-false, is a dictionary of field
331
+ // keys and error diagnostic strings.
332
+ connector.writeResource = function writeResource(resource, resID,
333
+ local creds, errors, error, data, change) {
334
+ // badField - test for a bad string
297
335
  function badField(str) { !string(str) || str.trim().length == 0 }
336
+
298
337
  creds = { urlName: resource.urlName }
299
338
  errors = { }
300
339
  if resource.label
@@ -303,6 +342,8 @@ closure psmsFsConnector(psm, api, local connClassID, connector, watch) {
303
342
  creds.label = resource.urlName
304
343
  if badField(creds.label)
305
344
  errors.label = "invalid label"
345
+ else if !resID && findResource({ label: creds.label })
346
+ errors.label = "duplicate name"
306
347
  if badField(resource.urlName)
307
348
  errors.urlName = "required field"
308
349
  if length(errors) > 0
@@ -312,14 +353,36 @@ closure psmsFsConnector(psm, api, local connClassID, connector, watch) {
312
353
  creds.hidden = resource.hidden
313
354
  if resource.locked
314
355
  creds.locked = resource.locked
315
- resID = hashResourceID(resource)
316
- `(error, data) = connector.vault.addResource(resID, creds)
356
+ if resID {
357
+ `(error, data) = connector.vault.updateResource(resID, creds)
358
+ change = { changed: [resID] }
359
+ } else {
360
+ resID = UUID()
361
+ `(error, data) = connector.vault.addResource(resID, creds)
362
+ change = { added: [resID] }
363
+ }
317
364
  if data {
318
- watch.notify(connClassID, { added: [resID] })
319
- data = resID } }
365
+ watch.notify(connClassID, change)
366
+ data = resID
367
+ }
368
+ }
320
369
  list(error, data)
321
370
  }
322
-
371
+
372
+ // addResource
373
+ //
374
+ // Add a new resource, returning a standard error tuple per writeResource.
375
+ connector.addResource = function addResource(resource) {
376
+ writeResource(resource)
377
+ }
378
+
379
+ // updateResource
380
+ //
381
+ // Update an existing resource, returning a standard error tuple per writeResource.
382
+ connector.updateResource = function updateResource(resID, resource) {
383
+ writeResource(resource, resID)
384
+ }
385
+
323
386
  // getResource
324
387
  //
325
388
  // Get credentials for a named resource, matching the dictionary semantics of addResource. The
@@ -6,7 +6,7 @@
6
6
  *
7
7
  * column positioning: // // !
8
8
  *
9
- * Copyright (c) 2019-2021 by Richard C. Zulch
9
+ * Copyright (c) 2019-2022 by Richard C. Zulch
10
10
  *
11
11
  */
12
12
 
@@ -267,15 +267,20 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
267
267
  //
268
268
  // applySubstitutions
269
269
  //
270
- // Apply substitutions from readSubstitutionFile to string, returning a new string.
270
+ // Apply substitutions from readSubstitutionFile to string, returning a tuple of the resulting
271
+ // text and the number of substitution patterns actually used from the supplied dictionary.
271
272
  //
272
- function applySubstitutions(text, subs) {
273
- text.replace(RegExp("[$][-A-Za-z0-9_./()\\\\]+[$]", "g"), function(match) {
274
- if subs[match.slice(1,-1)]
275
- { }
273
+ function applySubstitutions(text, subs, local subbed) {
274
+ subbed = { }
275
+ list(text.replace(RegExp("[$][-A-Za-z0-9_./()\\\\]+[$]", "g"), function(match, local subst) {
276
+ subst = subs[match.slice(1,-1)]
277
+ if subst {
278
+ subbed[match] = true
279
+ subst
280
+ }
276
281
  else
277
282
  match
278
- })
283
+ }), length(subbed))
279
284
  }
280
285
 
281
286
  //
@@ -573,7 +578,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
573
578
  //
574
579
  // Handling the catenates is a bit tricky. One set of buildrules can have multiple catentates,
575
580
  // distinguished by having a unique writepath derived from the destfile parameter. All groups
576
- // having the same writepath contribute to a single catenate, with the sources ordere first by
581
+ // having the same writepath contribute to a single catenate, with the sources ordered first by
577
582
  // group number (low to high) and then order in the source list, first to last. If any component
578
583
  // contributing to a catenate is out-of-date with respect to it then they are all built and
579
584
  // combined into the single output file. This computes a set of ordering integers for the parts
@@ -649,13 +654,13 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
649
654
  outfile = JSpath.join(outdirpath, group.destfile)
650
655
  else
651
656
  outfile = JSpath.join(outdirpath, source)
652
- fs.info(infile, function(error, instat, local listing, addgroup, file) {
657
+ fs.info(infile, false, function(error, instat, local listing, addgroup, file) {
653
658
  if error
654
659
  build.errorlist.push(error)
655
660
  else if instat.type != "directory"
656
661
  addEntry(infile, outfile, instat, sdex) // add execution entry if needed
657
662
  else { // add a new group for directory source
658
- `(error, listing) = fs.dirList(infile, { head: true })
663
+ `(error, listing) = fs.dirList(infile, { stat: true })
659
664
  if error
660
665
  build.errorlist.push(error)
661
666
  else {
@@ -754,19 +759,20 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
754
759
  entry.zip = true
755
760
  if !entry.writepath.toLowerCase().endsWith(".zip")
756
761
  entry.writepath = entry.writepath.concat(".zip") // always ends with .zip
762
+ outfile = entry.writepath
757
763
  }
758
764
  if entry.catenate || entry.zip {
759
765
  entry.catorder = group.groupno * sourcemax + sdex // order of groups then sources
760
766
  if build.testats[outfile] && instat.mtimeMs < build.testats[outfile].mtimeMs
761
767
  instat = build.testats[outfile] // make source at least as recent as template
762
- } else
768
+ } else if !entry.unzip
763
769
  entry.writepath = outfile
764
770
  ++pending.active
765
771
  if destcheck { // destcheck is special target for determining if task needed
766
772
  entry.destcheck = JSpath.join(outdirpath, destcheck.pop())
767
- fs.info(entry.destcheck, addIfOutOfDate)
773
+ fs.info(entry.destcheck, false, addIfOutOfDate)
768
774
  } else
769
- fs.info(outfile, addIfOutOfDate)
775
+ fs.info(outfile, false, addIfOutOfDate)
770
776
 
771
777
  function addIfOutOfDate(error, outstat, local entryop) {
772
778
  if error || toint(instat.mtimeMs) > toint(outstat.mtimeMs) {
@@ -861,9 +867,8 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
861
867
  if error
862
868
  build.errorlist.push(error)
863
869
  else if content.startsWith("$$") // prefix to cause version substitution
864
- build.templates[writepath] = applySubstitutions(content.slice(2), verFileDefs)
865
- else
866
- build.templates[writepath] = content
870
+ `(content) = applySubstitutions(content.slice(2), verFileDefs)
871
+ build.templates[writepath] = content
867
872
  }
868
873
  } ()
869
874
  }
@@ -890,13 +895,21 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
890
895
  `(error, result) = fs.unzip(entry.readpath, entry.writepath) // unzip source into destination
891
896
  if !error && entry.destcheck
892
897
  fs.touch(entry.destcheck) // show we completed
893
- } else if entry.copy
894
- `(error, result) = fs.copy(entry.readpath, entry.writepath, { overwrite: true }) // copy file from source into destination
898
+ } else if entry.copy {
899
+ `(error, result) = fs.util.deepcopy(entry.readpath, false, fs, dirpath, {
900
+ erase: true // not really needed
901
+ overwrite: true // overwrite files
902
+ force: true // ignore modify date
903
+ preserve: true // preserve attributes
904
+ })
905
+ if error // deepcopy returns a list of errors
906
+ error = error.0 // but we're only doing one item
907
+ }
895
908
  else if ((`(error, content) = fs.readFile(entry.readpath)) && !error) {
896
909
  if entry.license
897
- content = applySubstitutions(content, licFileDefs)
910
+ `(content) = applySubstitutions(content, licFileDefs)
898
911
  if entry.version
899
- content = applySubstitutions(content, verFileDefs)
912
+ `(content) = applySubstitutions(content, verFileDefs)
900
913
  if entry.naanpack
901
914
  `(error, content) = parseAndPack(content, entry.readpath, { naanpack: true })
902
915
  else if entry.jspack
@@ -962,11 +975,15 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
962
975
  docat = true
963
976
  }
964
977
  if docat { // catenate the files
965
- if template
966
- content = applySubstitutions(template, catsub)
978
+ if template {
979
+ `(content, result) = applySubstitutions(template, catsub)
980
+ if length(catsub) != result
981
+ build.errorlist.push(Error("Matched ", result, " of ", length(catsub),
982
+ "substitutions in template:", build.tepaths[writepath]))
983
+ }
967
984
  else
968
985
  content = catenates[writepath].map(function(entry){entry.content}).join("\n")
969
- if zip {
986
+ if dozip {
970
987
  zipmap = { }
971
988
  zipmap[JSpath.basename(writepath)] = content
972
989
  `(error, content) = zip(zipmap) // compress the catenated file