@voxgig/sdkgen 4.32.1 → 4.34.0

package/project/.sdk/tm/zig/feature/proxy.zig

@@ -83,12 +83,22 @@ pub const ProxyFeature = struct {
         }
         const no_proxy: []const []const u8 = np_list.toOwnedSlice(h.A()) catch &.{};
 
-        self.track.url = url;
+        const util = ctx.util();
+
+        // A proxy URL may carry credentials as userinfo, from the option or
+        // the environment, and neither is under a sensitive key name.
+        for (userinfo_parts(url)) |part| {
+            util.clean_add(ctx, part);
+            util.clean_add(ctx, percent_decode(part));
+        }
+
+        // The tracked url is what a caller reads back, so it is cleaned; the
+        // wrapper routes with the raw one.
+        self.track.url = util.clean_str(ctx, url);
         self.track.no_proxy = no_proxy;
 
-        const util = ctx.util();
         const w = h.A().create(WrapCtx) catch unreachable;
-        w.* = .{ .inner = util.fetcher, .track = self.track };
+        w.* = .{ .inner = util.fetcher, .track = self.track, .url = url };
         util.fetcher = .{ .ctx = @ptrCast(w), .call = wrapCall };
     }
     fn vdispatch(p: *anyopaque, name: []const u8, ctx: *Context) void {
@@ -113,14 +123,46 @@ pub const ProxyFeature = struct {
 const WrapCtx = struct {
     inner: Fetcher,
     track: *ProxyTrack,
+    url: []const u8,
 };
 
 fn wrapCall(p: *anyopaque, ctx: *Context, url: []const u8, fetchdef: Value) err.E!Value {
     const w: *WrapCtx = @ptrCast(@alignCast(p));
-    const routed = route(w.track, url, fetchdef);
+    const routed = route(w.url, w.track, url, fetchdef);
     return w.inner.invoke(ctx, url, routed);
 }
 
+// The user and password of `scheme://user:pass@host`, raw as written.
+fn userinfo_parts(url: []const u8) []const []const u8 {
+    var out: std.ArrayList([]const u8) = .empty;
+    const start = if (std.mem.indexOf(u8, url, "://")) |i| i + 3 else return &.{};
+    const rest = url[start..];
+    const at = std.mem.indexOfScalar(u8, rest, '@') orelse return &.{};
+    const userinfo = rest[0..at];
+    if (std.mem.indexOfScalar(u8, userinfo, '/') != null) return &.{};
+    var it = std.mem.splitScalar(u8, userinfo, ':');
+    while (it.next()) |part| {
+        if (part.len != 0) out.append(h.A(), part) catch {};
+    }
+    return out.toOwnedSlice(h.A()) catch &.{};
+}
+
+fn percent_decode(s: []const u8) []const u8 {
+    var out: std.ArrayList(u8) = .empty;
+    var i: usize = 0;
+    while (i < s.len) : (i += 1) {
+        if (s[i] == '%' and i + 3 <= s.len) {
+            if (std.fmt.parseInt(u8, s[i + 1 .. i + 3], 16)) |byte| {
+                out.append(h.A(), byte) catch {};
+                i += 2;
+                continue;
+            } else |_| {}
+        }
+        out.append(h.A(), s[i]) catch {};
+    }
+    return out.toOwnedSlice(h.A()) catch s;
+}
+
 // Zig 0.16 has no free-standing environment lookup: the block the process was
 // started with is owned by an Io implementation, and library code is expected
 // to be handed one. The SDK's Io is the process-global singleton (core/mem.zig),
@@ -165,8 +207,7 @@ fn bypass(no_proxy: []const []const u8, url: []const u8) bool {
     return false;
 }
 
-fn route(track: *ProxyTrack, url: []const u8, fetchdef: Value) Value {
-    const proxy_url = track.url;
+fn route(proxy_url: []const u8, track: *ProxyTrack, url: []const u8, fetchdef: Value) Value {
     const no_proxy = track.no_proxy;
 
     if (proxy_url.len == 0 or bypass(no_proxy, url)) {

package/project/.sdk/tm/zig/feature/secrets.zig

@@ -300,6 +300,10 @@ pub const SecretsFeature = struct {
         var specs: std.ArrayList(sekreto.ProviderSpec) = .empty;
 
         if (0 != explicit.len) {
+            // Every value this feature resolves or buys is a secret the SDK
+            // handles, and none arrives under an option key the intake
+            // registration saw.
+            util.clean_add(ctx, explicit);
             const keyed = sekreto.envkey(alloc, st.secretname, "") catch
                 sekreto.Answer([]const u8){ .err = "secrets: out of memory" };
             switch (keyed) {
@@ -572,6 +576,7 @@ fn resolve_once(st: *State, ctx: *Context) E!void {
         .err => |message| return ctx.fail(ERRCODE, h.A().dupe(u8, message) catch message),
         .ok => |v| v,
     };
+    if (value) |v| ctx.util().clean_add(ctx, v);
 
     if (st.exchange == null) {
         // An UNCACHED miss after an earlier hit is a revocation: the chain
@@ -793,6 +798,8 @@ fn buy(st: *State, ctx: *Context) E![]const u8 {
         ) catch "secrets: token exchange returned no token");
     }
 
+    ctx.util().clean_add(ctx, token);
+
     st.cred = token;
     return token;
 }

package/project/.sdk/tm/zig/feature/telemetry.zig

@@ -72,12 +72,14 @@ pub const TelemetryFeature = struct {
         h.setp(span, "durationMs", h.vnum(dur));
         h.setp(span, "ok", h.vbool(ok));
 
+        const out = ctx.util().clean(ctx, span);
+
         self.active_spans -= 1;
-        self.spans.append(h.A(), span) catch {};
+        self.spans.append(h.A(), out) catch {};
 
         const exporter = h.getp(self.options, "exporter");
         if (exporter == .function) {
-            _ = h.call_vfn(exporter, span);
+            _ = h.call_vfn(exporter, out);
         }
     }
 

package/project/.sdk/tm/zig/test/feature_test.zig

@@ -697,7 +697,7 @@ test "feature debug: redacts and honours onentry max" {
     try testing.expect(df.entries.items.len == 1);
     try testing.expect(seen.len() == 2);
     const hdrs = h.getp(seen.at(0), "headers");
-    try testing.expect(h.veq(h.getp(hdrs, "authorization"), h.vstr("<redacted>")));
+    try testing.expect(h.veq(h.getp(hdrs, "authorization"), h.vstr("[redacted]")));
 }
 
 test "feature debug: captures failures" {
@@ -728,7 +728,7 @@ test "feature debug: injected clock and custom redact" {
         .{ "x-ok", h.vstr("show") },
     }) });
     const hdrs = h.getp(df.entries.items[0], "headers");
-    try testing.expect(h.veq(h.getp(hdrs, "x-secret"), h.vstr("<redacted>")));
+    try testing.expect(h.veq(h.getp(hdrs, "x-secret"), h.vstr("[redacted]")));
     try testing.expect(h.veq(h.getp(hdrs, "x-ok"), h.vstr("show")));
 }
 

package/project/sdkgen-package.json

@@ -3,7 +3,7 @@
     "package": 1
   },
   "name": "@voxgig/sdkgen",
-  "version": "4.32.1",
+  "version": "4.34.0",
   "provides": {
     "target": [
       "c",

package/src/action/copies.ts

@@ -0,0 +1,219 @@
+import Path from 'node:path'
+
+import { createHash } from 'node:crypto'
+
+import type { ActionContext } from '../types'
+
+import {
+  definitionFileName, definitionFolder, indexName,
+} from '../helpers/definition'
+
+import { KINDS } from './kind'
+
+import type { Source } from './resolve'
+
+
+// What every add last wrote into `.sdk`, by fingerprint, and the version of
+// the package each item came from. It is the only record that tells a copy
+// the project changed from one its source has since moved past.
+const COPIES = 'sdkgen-copies.json'
+
+const ABOUT = 'Written by voxgig-sdkgen on every add: what it copied into ' +
+  '.sdk, so that doctor, generate and package update can tell an outdated ' +
+  'copy from a local edit. Commit it; do not edit it.'
+
+
+type CopyItem = {
+  package?: string
+  version?: string
+}
+
+type CopyRecord = {
+  items: Record<string, CopyItem>
+  files: Record<string, string>
+}
+
+
+function copiesPath(folder: string): string {
+  return Path.join(folder, COPIES)
+}
+
+
+function itemKey(kind: string, name: string): string {
+  return kind + '/' + name
+}
+
+
+// CRLF and LF copies of one file share a fingerprint, as doctor compares
+// them. latin1 maps each byte to one character, so nothing else changes.
+function fingerprint(content: any): string {
+  const bytes = Buffer.isBuffer(content) ? content : Buffer.from(String(content))
+  const lf = Buffer.from(bytes.toString('latin1').replace(/\r\n/g, '\n'), 'latin1')
+  return createHash('sha256').update(lf).digest('hex').slice(0, 16)
+}
+
+
+// An unreadable record is treated as absent: every copy is then unrecorded,
+// which is exactly the state of a project from before the record existed.
+function readCopies(fs: any, folder: string): CopyRecord {
+  const empty: CopyRecord = { items: {}, files: {} }
+  const path = copiesPath(folder)
+
+  if (!fs.existsSync(path)) {
+    return empty
+  }
+
+  try {
+    const read = JSON.parse(String(fs.readFileSync(path, 'utf8')))
+    return {
+      items: { ...(read?.items ?? {}) },
+      files: { ...(read?.files ?? {}) },
+    }
+  }
+  catch (err: any) {
+    return empty
+  }
+}
+
+
+function writeCopies(fs: any, folder: string, record: CopyRecord) {
+  const sorted = (obj: Record<string, any>) => Object.fromEntries(
+    Object.keys(obj).sort().map((key: string) => [key, obj[key]]))
+
+  fs.writeFileSync(copiesPath(folder), JSON.stringify({
+    about: ABOUT,
+    items: sorted(record.items),
+    files: sorted(record.files),
+  }, null, 2) + '\n')
+}
+
+
+function provenanceOf(source: Source): CopyItem {
+  return {
+    ...(null == source.package ? {} : { package: source.package }),
+    ...(null == source.version ? {} : { version: source.version }),
+  }
+}
+
+
+// The trees and definition files an add owns. An index is excluded: every
+// add rewrites it and no source holds a copy to compare it with.
+const TREE_ROOTS = Array.from(new Set(Object.values(KINDS)
+  .flatMap((kind: any) => (kind.trees ?? [])
+    .map((tree: any) => String(tree.path).split('{name}')[0]))))
+
+function isCopy(rel: string): boolean {
+  if (TREE_ROOTS.some((root: string) => rel.startsWith(root))) {
+    return true
+  }
+
+  const dir = Path.posix.dirname(rel)
+  const base = Path.posix.basename(rel)
+
+  return Object.keys(KINDS).some((kind: string) =>
+    dir === definitionFolder('', kind).split(Path.sep).join('/') &&
+    base !== indexName(kind) &&
+    base.endsWith(definitionFileName('')))
+}
+
+
+function relativeTo(folder: string, abs: string): string | undefined {
+  const rel = Path.relative(Path.resolve(folder), Path.resolve(abs))
+    .split(Path.sep).join('/')
+
+  return ('' === rel || rel.startsWith('../') || Path.isAbsolute(rel)) ?
+    undefined : rel
+}
+
+
+// Called by each add with what its own jostraca run wrote. A dry run writes
+// nothing, so it records nothing.
+function recordCopies(
+  actx: ActionContext, jres: any, kind: string, sources: Source[],
+) {
+  if (true === actx.opts?.dryrun || null == jres?.files) {
+    return
+  }
+
+  const fs = actx.fs()
+  const folder = actx.folder
+  const record = readCopies(fs, folder)
+
+  const files: string[] = [
+    ...(jres.files.written ?? []), ...(jres.files.unchanged ?? []),
+  ]
+
+  for (const abs of files) {
+    const rel = relativeTo(folder, abs)
+
+    if (null != rel && isCopy(rel) && fs.existsSync(abs)) {
+      record.files[rel] = fingerprint(fs.readFileSync(abs))
+    }
+  }
+
+  for (const source of sources) {
+    record.items[itemKey(kind, source.name)] = provenanceOf(source)
+  }
+
+  pruneMissing(fs, folder, record)
+  writeCopies(fs, folder, record)
+}
+
+
+// After a remove: the item goes, with every file entry nothing holds now.
+function forgetCopies(actx: ActionContext, kind: string, name: string) {
+  const fs = actx.fs()
+  const folder = actx.folder
+
+  if (true === actx.opts?.dryrun || !fs.existsSync(copiesPath(folder))) {
+    return
+  }
+
+  const record = readCopies(fs, folder)
+  delete record.items[itemKey(kind, name)]
+  pruneMissing(fs, folder, record)
+  writeCopies(fs, folder, record)
+}
+
+
+function pruneMissing(fs: any, folder: string, record: CopyRecord) {
+  for (const rel of Object.keys(record.files)) {
+    if (!fs.existsSync(Path.join(folder, ...rel.split('/')))) {
+      delete record.files[rel]
+    }
+  }
+}
+
+
+// Was this project file left exactly as an add wrote it?
+function untouched(
+  fs: any, folder: string, record: CopyRecord, rel: string,
+): boolean | undefined {
+  const recorded = record.files[rel]
+
+  if (null == recorded) {
+    return undefined
+  }
+
+  const abs = Path.join(folder, ...rel.split('/'))
+
+  return fs.existsSync(abs) && recorded === fingerprint(fs.readFileSync(abs))
+}
+
+
+export type {
+  CopyItem,
+  CopyRecord,
+}
+
+export {
+  COPIES,
+  copiesPath,
+  itemKey,
+  fingerprint,
+  readCopies,
+  recordCopies,
+  forgetCopies,
+  untouched,
+  isCopy,
+}