depyo 1.0.1 → 1.0.3

package/README.md

@@ -32,6 +32,9 @@ node depyo.js --out /path/to/file.pyc
 # Marshal-only blob (no .pyc header)
 node depyo.js --marshal --py-version 3.11 /path/to/blob.bin
 node depyo.js --marshal /path/to/blob.bin
+
+# Fast marshal scan (no decompile)
+node depyo.js --marshal-scan /path/to/blob.bin
 ```
 Without `--py-version`, depyo scans supported versions (oldest → newest) and accepts the first clean output when all clean candidates agree. If outputs diverge (ambiguous), it stops and asks for `--py-version`. Use `--debug` to see scan results.
 
@@ -45,6 +48,7 @@ Without `--py-version`, depyo scans supported versions (oldest → newest) and a
 - `--skip-path` flatten output paths (write next to input)
 - `--out` print source to stdout instead of files
 - `--marshal` treat input as raw marshalled data (no .pyc header, auto-scan versions)
+- `--marshal-scan` fast scan marshal blobs and print version candidates
 - `--py-version <x.y>` bytecode version hint (use with `--marshal`)
 - `--basedir <dir>` override output root (default: alongside input)
 - `--file-ext <ext>` change emitted extension (default `py`)

package/depyo.js

@@ -21,6 +21,8 @@ global.g_cliArgs = {
     skipPath: false,
     sendToStdout: false,
     marshal: false,
+    marshalScan: false,
+    strict: false,
     pyVersion: null,
     silent: false,
     fileExt: 'py',
@@ -28,11 +30,14 @@ global.g_cliArgs = {
     filenames: []
 };
 
+let g_dirtyFiles = [];   // Files where decompiler caught at least one opcode exception.
+
 let g_totalInThroughput = 0;
 let g_totalOutThroughput = 0;
 let g_totalExecTime = 0;
 let g_totalFiles = 0;
 let g_pyVersionInfo = null;
+let g_marshalScanStats = {ok: 0, ambiguous: 0, failed: 0};
 
 function printUsage() {
     console.log(`Usage: node depyo.js [options] <file.pyc|archive.zip> [...]
@@ -48,6 +53,8 @@ Options:
   --skip-path         Flatten output paths (write files next to inputs)
   --out               Print decompiled source to stdout instead of files
   --marshal           Treat input as raw marshalled data (no .pyc header)
+  --marshal-scan      Fast scan of marshal blobs (no decompile, prints version)
+  --strict            Re-throw on first opcode handler exception (default: log + continue + non-zero exit)
   --py-version <x.y>  Python bytecode version hint (auto-scan if omitted)
   --basedir <path>    Output base directory (default: alongside input)
   --file-ext <ext>    Extension for generated source (default: py)
@@ -77,6 +84,11 @@ function parseCLIParams() {
             g_cliArgs.sendToStdout = true;
         } else if (cliParam.toLowerCase() == "--marshal") {
             g_cliArgs.marshal = true;
+        } else if (cliParam.toLowerCase() == "--marshal-scan" || cliParam.toLowerCase() == "--marshal-smoke") {
+            g_cliArgs.marshalScan = true;
+            g_cliArgs.marshal = true;
+        } else if (cliParam.toLowerCase() == "--strict") {
+            g_cliArgs.strict = true;
         } else if (cliParam.toLowerCase() == "--py-version") {
             g_cliArgs.pyVersion = process.argv[++idx];
         } else if (cliParam.toLowerCase() == "--basedir") {
@@ -103,6 +115,44 @@ function normalizeMarshalOutput(src) {
         .trim();
 }
 
+function scanMarshalBuffer(buffer, filenameLabel) {
+    if (g_pyVersionInfo) {
+        const trial = PycReader.TryParseMarshal(buffer, g_pyVersionInfo);
+        if (!trial) {
+            g_marshalScanStats.failed++;
+            console.log(`${filenameLabel}: no parse with ${g_pyVersionInfo.major}.${g_pyVersionInfo.minor}`);
+            return;
+        }
+        g_marshalScanStats.ok++;
+        console.log(`${filenameLabel}: forced ${g_pyVersionInfo.major}.${g_pyVersionInfo.minor} unknown=${trial.unknown}/${trial.total} remaining=${trial.remaining}`);
+        return;
+    }
+
+    const results = PycReader.ScanMarshalCandidates(buffer);
+    if (!results.length) {
+        g_marshalScanStats.failed++;
+        console.log(`${filenameLabel}: no candidates`);
+        return;
+    }
+
+    const best = results[0];
+    const ambiguous = results.filter(r =>
+        r.unknown === best.unknown &&
+        r.remaining === best.remaining &&
+        r.unknownRatio === best.unknownRatio
+    );
+
+    if (ambiguous.length > 1) {
+        g_marshalScanStats.ambiguous++;
+        const versions = ambiguous.map(r => `${r.versionInfo.major}.${r.versionInfo.minor}`).join(', ');
+        console.log(`${filenameLabel}: ambiguous candidates (${versions})`);
+        return;
+    }
+
+    g_marshalScanStats.ok++;
+    console.log(`${filenameLabel}: best=${best.versionInfo.major}.${best.versionInfo.minor} unknown=${best.unknown}/${best.total} remaining=${best.remaining}`);
+}
+
 function attemptMarshalDecompile(buffer, versionInfo, opts = {}) {
     const prevSilent = g_cliArgs.silent;
     const prevDebug = g_cliArgs.debug;
@@ -195,6 +245,11 @@ function decompilePycObject(data) {
         if (!Buffer.isBuffer(buffer)) {
             buffer = fs.readFileSync(data);
         }
+        if (g_cliArgs.marshalScan) {
+            const label = typeof data === 'string' ? data : '<buffer>';
+            scanMarshalBuffer(buffer, label);
+            return;
+        }
         let rdr = null;
         let pySrc = null;
         let genSecs = 0;
@@ -292,9 +347,33 @@ function decompilePycObject(data) {
             let genStartTS = process.hrtime.bigint();
             let decompiler = new PycDecompiler(obj);
             let ast = decompiler.decompile();
-            let pycResult = ast.codeFragment();
-            pySrc = pycResult.toString();
+            let renderError = null;
+            try {
+                let pycResult = ast.codeFragment();
+                pySrc = pycResult.toString();
+            } catch (ex) {
+                if (g_cliArgs.strict) throw ex;
+                renderError = ex;
+                decompiler.errors.push({
+                    opcode: 'RENDER',
+                    codeObject: obj?.Name?.toString?.() || '<root>',
+                    message: ex.message,
+                    stack: ex.stack
+                });
+                decompiler.cleanBuild = false;
+                pySrc = `# DECOMPILER ERROR: codeFragment() threw: ${ex.message}\n`;
+                if (!g_cliArgs.silent) {
+                    console.error(`RENDER EXCEPTION in '${obj?.Name}': ${ex.message}`);
+                    if (g_cliArgs.debug) console.error(ex.stack);
+                }
+            }
             genSecs = Number(process.hrtime.bigint() - genStartTS) / 1000000000;
+            if (!decompiler.cleanBuild) {
+                g_dirtyFiles.push({
+                    file: typeof data === 'string' ? data : (obj?.FileName || '<buffer>'),
+                    errors: decompiler.errors.length
+                });
+            }
         }
         if (!pySrc.endsWith("\n")) {
             pySrc += "\n";
@@ -369,8 +448,30 @@ g_baseDir = Path.resolve(baseInputDir, 'decompiled') + '/';
 
 DecompileModule(g_cliArgs.filenames);
 
+if (g_cliArgs.marshalScan) {
+    console.log(`Marshal scan summary: ok=${g_marshalScanStats.ok}, ambiguous=${g_marshalScanStats.ambiguous}, failed=${g_marshalScanStats.failed}`);
+    if (g_marshalScanStats.failed > 0) {
+        process.exit(1);
+    }
+    if (g_marshalScanStats.ambiguous > 0) {
+        process.exit(2);
+    }
+    process.exit(0);
+}
+
 if (!g_cliArgs.sendToStdout) {
     const inRate = (g_totalInThroughput / g_totalExecTime).toFixed(2);
     const outRate = (g_totalOutThroughput / g_totalExecTime).toFixed(2);
     console.log(`Processed ${g_totalFiles} files in ${g_totalExecTime.toFixed(3)}s. In: ${g_totalInThroughput} bytes (${inRate} B/s). Out: ${g_totalOutThroughput} bytes (${outRate} B/s).`);
 }
+
+if (g_dirtyFiles.length > 0) {
+    console.error(`\nDirty decompile: ${g_dirtyFiles.length} file(s) had handler exceptions (output may be partial):`);
+    for (const d of g_dirtyFiles.slice(0, 20)) {
+        console.error(`  - ${d.file}  (${d.errors} opcode error${d.errors === 1 ? '' : 's'})`);
+    }
+    if (g_dirtyFiles.length > 20) {
+        console.error(`  ... and ${g_dirtyFiles.length - 20} more`);
+    }
+    process.exit(1);
+}

package/lib/OpCodes.js

@@ -329,6 +329,9 @@ class OpCodes
     static LOAD_ZERO_SUPER_ATTR_A = 319;           // Python 3.13+ zero-cost super attr
     static LOAD_ZERO_SUPER_METHOD_A = 320;         // Python 3.13+ zero-cost super method
 
+    // Python 3.15 new opcodes
+    static TRACE_RECORD_A = 321;                   // Python 3.15 ->           trace recording (ignore)
+
 
     // enum cmp_op
     // {
@@ -391,7 +394,7 @@ class OpCodes
 
         if (reader.versionCompare(3, 6) >= 0) {
             while (opCodeID == OpCodes.EXTENDED_ARG_A) {
-                argument = argument | code[++opOffset] << 8;
+                argument = (argument << 8) | code[++opOffset];
                 opCodeID = this.GetOpCodeID(code, ++opOffset);
 
                 // Break if we hit end of bytecode
@@ -401,9 +404,15 @@ class OpCodes
             }
             argument <<= 8;
         } else {
+            // Pre-3.6: EXTENDED_ARG carries a 16-bit operand that becomes
+            // the upper 16 bits of the next instruction's argument. After
+            // reading both operand bytes we must advance opOffset one more
+            // step so the caller lands on the real opcode (not the trailing
+            // operand byte of EXTENDED_ARG).
             if (opCodeID == OpCodes.EXTENDED_ARG_A) {
                 argument = code[++opOffset] | code[++opOffset] << 8;
                 argument <<= 16;
+                opOffset++;
             }
         }
 
@@ -522,10 +531,18 @@ class OpCodes
                         opCode.Name = this.CodeObject.Names.Value[opCode.Argument].toString();
                     }
                 } else if (opCode.HasFree) {
-                    if (opCode.Argument < this.CodeObject.CellVars.Value.length) {
-                        opCode.FreeName = this.CodeObject.CellVars.Value[opCode.Argument].toString();
-                    } else if ((opCode.Argument - this.CodeObject.CellVars.Value.length) < this.CodeObject.FreeVars.Value.length) {
-                        opCode.FreeName = this.CodeObject.FreeVars.Value[opCode.Argument - this.CodeObject.CellVars.Value.length].toString();
+                    // 3.11+ stores cells/frees inside localsplus, so the opcode argument
+                    // is an index into [locals | cells | frees]. Strip the locals prefix
+                    // before looking up into the split CellVars/FreeVars tuples.
+                    const isNewLayout = this.CodeObject.Reader?.versionCompare?.(3, 11) >= 0;
+                    const localsLen = isNewLayout ? (this.CodeObject.VarNames?.Value?.length ?? 0) : 0;
+                    const freeIdx = opCode.Argument - localsLen;
+                    const cellLen = this.CodeObject.CellVars?.Value?.length ?? 0;
+                    const freeLen = this.CodeObject.FreeVars?.Value?.length ?? 0;
+                    if (freeIdx >= 0 && freeIdx < cellLen) {
+                        opCode.FreeName = this.CodeObject.CellVars.Value[freeIdx].toString();
+                    } else if (freeIdx >= cellLen && (freeIdx - cellLen) < freeLen) {
+                        opCode.FreeName = this.CodeObject.FreeVars.Value[freeIdx - cellLen].toString();
                     } else {
                         opCode.FreeName = `##FREEVAR_${opCode.Argument}##`;
                     }