depyo 1.0.0 → 1.0.1

package/README.md

@@ -28,7 +28,12 @@ node depyo.js --skip-path /path/to/file.pyc
 
 # Dump to stdout instead of files
 node depyo.js --out /path/to/file.pyc
+
+# Marshal-only blob (no .pyc header)
+node depyo.js --marshal --py-version 3.11 /path/to/blob.bin
+node depyo.js --marshal /path/to/blob.bin
 ```
+Without `--py-version`, depyo scans supported versions (oldest → newest) and accepts the first clean output when all clean candidates agree. If outputs diverge (ambiguous), it stops and asks for `--py-version`. Use `--debug` to see scan results.
 
 ### CLI options
 - `--asm` emit `.pyasm` disassembly alongside source
@@ -39,6 +44,8 @@ node depyo.js --out /path/to/file.pyc
 - `--skip-source-gen` skip writing `.py` (use with `--asm/--dump`)
 - `--skip-path` flatten output paths (write next to input)
 - `--out` print source to stdout instead of files
+- `--marshal` treat input as raw marshalled data (no .pyc header, auto-scan versions)
+- `--py-version <x.y>` bytecode version hint (use with `--marshal`)
 - `--basedir <dir>` override output root (default: alongside input)
 - `--file-ext <ext>` change emitted extension (default `py`)
 
@@ -58,6 +65,14 @@ node depyo.js --out /path/to/file.pyc
   node scripts/run-matrix.js                  # full sweep
   node scripts/run-matrix.js --pattern py311_exception_groups --fail-fast
   ```
+- Marshal fixtures (headerless marshal blobs):
+  ```bash
+  node scripts/run-marshal-fixtures.js
+  ```
+- Regenerate marshal fixtures:
+  ```bash
+  node scripts/generate-marshal-fixtures.js --clean
+  ```
 - Modern fixtures are generated via `test/generate_modern_tests.py` (Python 3.8+ on PATH).
 
 ## Support matrix

package/depyo.js

@@ -20,6 +20,9 @@ global.g_cliArgs = {
     skipSource: false,
     skipPath: false,
     sendToStdout: false,
+    marshal: false,
+    pyVersion: null,
+    silent: false,
     fileExt: 'py',
     baseDir: null,
     filenames: []
@@ -29,6 +32,7 @@ let g_totalInThroughput = 0;
 let g_totalOutThroughput = 0;
 let g_totalExecTime = 0;
 let g_totalFiles = 0;
+let g_pyVersionInfo = null;
 
 function printUsage() {
     console.log(`Usage: node depyo.js [options] <file.pyc|archive.zip> [...]
@@ -43,6 +47,8 @@ Options:
   --skip-source-gen   Do not emit .py source (useful with --asm/--dump)
   --skip-path         Flatten output paths (write files next to inputs)
   --out               Print decompiled source to stdout instead of files
+  --marshal           Treat input as raw marshalled data (no .pyc header)
+  --py-version <x.y>  Python bytecode version hint (auto-scan if omitted)
   --basedir <path>    Output base directory (default: alongside input)
   --file-ext <ext>    Extension for generated source (default: py)
 `);
@@ -69,6 +75,10 @@ function parseCLIParams() {
             g_cliArgs.skipPath = true;
         } else if (cliParam.toLowerCase() == "--out") {
             g_cliArgs.sendToStdout = true;
+        } else if (cliParam.toLowerCase() == "--marshal") {
+            g_cliArgs.marshal = true;
+        } else if (cliParam.toLowerCase() == "--py-version") {
+            g_cliArgs.pyVersion = process.argv[++idx];
         } else if (cliParam.toLowerCase() == "--basedir") {
             g_cliArgs.baseDir = process.argv[++idx];
         } else if (cliParam.toLowerCase() == "--file-ext") {
@@ -84,37 +94,176 @@ function parseCLIParams() {
     }
 }
 
+function normalizeMarshalOutput(src) {
+    return src
+        .replace(/\r\n/g, '\n')
+        .replace(/[ \t]+$/gm, '')
+        .replace(/\n{3,}/g, '\n\n')
+        .replace(/#[^\n]*$/gm, '')
+        .trim();
+}
+
+function attemptMarshalDecompile(buffer, versionInfo, opts = {}) {
+    const prevSilent = g_cliArgs.silent;
+    const prevDebug = g_cliArgs.debug;
+    if (opts.silent) {
+        g_cliArgs.silent = true;
+        g_cliArgs.debug = false;
+    }
+    try {
+        const rdr = new PycReader(buffer, {marshal: true, versionInfo});
+        const obj = rdr.ReadObject();
+        if (!obj || obj.ClassName !== "Py_CodeObject") {
+            return null;
+        }
+        const opcodes = new versionInfo.opcode(obj);
+        const {unknown, total} = PycReader.CountUnknownOpcodes(obj, rdr, opcodes.OpCodeList);
+        const unknownRatio = total > 0 ? unknown / total : 1;
+        const remaining = rdr.m_rdr.Reader.length - rdr.m_rdr.pc;
+
+        const genStartTS = process.hrtime.bigint();
+        const decompiler = new PycDecompiler(obj);
+        const ast = decompiler.decompile();
+        const pycResult = ast.codeFragment();
+        const pySrc = pycResult.toString();
+        const genSecs = Number(process.hrtime.bigint() - genStartTS) / 1000000000;
+
+        return {
+            reader: rdr,
+            obj,
+            pySrc,
+            genSecs,
+            cleanBuild: decompiler.cleanBuild,
+            unknown,
+            total,
+            unknownRatio,
+            remaining,
+            versionInfo
+        };
+    } catch (ex) {
+        if (opts.debug) {
+            console.log(`Marshal decompile failed for ${versionInfo.major}.${versionInfo.minor}: ${ex.message}`);
+        }
+        return null;
+    } finally {
+        g_cliArgs.silent = prevSilent;
+        g_cliArgs.debug = prevDebug;
+    }
+}
+
+function selectMarshalCandidate(buffer) {
+    const candidates = PycReader.ListSupportedVersions(false);
+    const attempts = [];
+    const cleanCandidates = [];
+    let baselineOutput = null;
+    let outputsDiverged = false;
+    for (const candidate of candidates) {
+        const result = attemptMarshalDecompile(buffer, candidate, {silent: true});
+        if (!result) {
+            continue;
+        }
+        attempts.push(result);
+        const isWorking = result.cleanBuild && result.unknown === 0 && result.remaining === 0;
+        if (isWorking) {
+            const normalized = normalizeMarshalOutput(result.pySrc || '');
+            cleanCandidates.push({...result, normalized});
+            if (baselineOutput === null) {
+                baselineOutput = normalized;
+            } else if (baselineOutput !== normalized) {
+                outputsDiverged = true;
+            }
+        }
+    }
+
+    if (!cleanCandidates.length) {
+        return attempts.length ? {best: null, attempts, ambiguous: false} : null;
+    }
+
+    if (outputsDiverged) {
+        return {best: null, attempts, ambiguous: true, cleanCandidates};
+    }
+
+    return {best: cleanCandidates[0], attempts, ambiguous: false};
+}
+
 function decompilePycObject(data) {
     try
     {
         let filename = null, obj = null;
         let startTS = process.hrtime.bigint();
-        let rdr = new PycReader(data);
-        try {
-            obj = rdr.ReadObject();
-            filename = g_baseDir + obj.FileName;
-        } catch (ex) {
-            if (ex instanceof PycReader.LoadError) {
-                // Save the binary file if it not already exists for future manual analysis.
-                if (!ex.FileName) {
-                    if (global.g_cliArgs?.debug) {
-                        console.log(`LoadError: ${ex.message} at position ${ex.position}`);
+        let buffer = data;
+        if (!Buffer.isBuffer(buffer)) {
+            buffer = fs.readFileSync(data);
+        }
+        let rdr = null;
+        let pySrc = null;
+        let genSecs = 0;
+
+        if (g_cliArgs.marshal) {
+            let attemptResult = null;
+            if (g_pyVersionInfo) {
+                attemptResult = attemptMarshalDecompile(buffer, g_pyVersionInfo);
+            } else {
+                const scan = selectMarshalCandidate(buffer);
+                attemptResult = scan ? scan.best : null;
+                if (g_cliArgs.debug && scan?.attempts?.length) {
+                    console.log("Marshal scan results:");
+                    for (const attempt of scan.attempts) {
+                        const info = attempt.versionInfo;
+                        console.log(`  ${info.major}.${info.minor}: clean=${attempt.cleanBuild} unknown=${attempt.unknown}/${attempt.total} remaining=${attempt.remaining}`);
+                    }
+                    if (attemptResult) {
+                        const info = attemptResult.versionInfo;
+                        console.log(`Selected marshal version: ${info.major}.${info.minor}`);
                     }
-                    return;
                 }
-                filename = g_baseDir + ex.FileName;
-                let dirPath =  Path.dirname(filename);
-                if (!g_cliArgs.skipPath && !fs.existsSync(dirPath)) {
-                    fs.mkdirSync(dirPath, {recursive: true});
+                if (!attemptResult && scan?.ambiguous) {
+                    const versions = scan.cleanCandidates
+                        ? scan.cleanCandidates.map(c => `${c.versionInfo.major}.${c.versionInfo.minor}`).join(', ')
+                        : 'unknown';
+                    throw new Error(`Ambiguous marshal version (${versions}). Provide --py-version X.Y to force.`);
                 }
-                let filenamePyc = filename.substring(0, filename.lastIndexOf('.')) + ".pyc";
-                fs.writeFileSync(filenamePyc, rdr.Reader);
-                console.log(`Error: ${ex.message}\nFile: ${filenamePyc}\nPosition: ${ex.position}`);
+            }
+
+            if (!attemptResult) {
+                throw new Error("No clean marshal candidate found. Provide --py-version X.Y to force.");
+            }
+            rdr = attemptResult.reader;
+            obj = attemptResult.obj;
+            pySrc = attemptResult.pySrc;
+            genSecs = attemptResult.genSecs;
+        } else {
+            rdr = new PycReader(buffer);
+            try {
+                obj = rdr.ReadObject();
+            } catch (ex) {
+                if (ex instanceof PycReader.LoadError) {
+                    // Save the binary file if it not already exists for future manual analysis.
+                    if (!ex.FileName) {
+                        if (global.g_cliArgs?.debug) {
+                            console.log(`LoadError: ${ex.message} at position ${ex.position}`);
+                        }
+                        return;
+                    }
+                    filename = g_baseDir + ex.FileName;
+                    let dirPath =  Path.dirname(filename);
+                    if (!g_cliArgs.skipPath && !fs.existsSync(dirPath)) {
+                        fs.mkdirSync(dirPath, {recursive: true});
+                    }
+                    let filenamePyc = filename.substring(0, filename.lastIndexOf('.')) + ".pyc";
+                    fs.writeFileSync(filenamePyc, rdr.Reader);
+                    console.log(`Error: ${ex.message}\nFile: ${filenamePyc}\nPosition: ${ex.position}`);
+                    return;
+                }
+                console.log(`Error: ${ex.message}\nStack:\n${ex.stacktrace}`);
                 return;
             }
-            console.log(`Error: ${ex.message}\nStack:\n${ex.stacktrace}`);
+        }
+
+        if (!obj) {
             return;
         }
+        filename = g_baseDir + obj.FileName;
 
         if (!g_cliArgs.sendToStdout) {
             console.log(`Processing ${filename}...`);
@@ -139,15 +288,18 @@ function decompilePycObject(data) {
                 fs.writeFileSync(filenameBase + ".pyasm", PycDisassembler.Disassemble(rdr, obj));
             }
         }
-        let genStartTS = process.hrtime.bigint();
-        let decompiler = new PycDecompiler(obj);
-        let ast = decompiler.decompile();
-        let pycResult = ast.codeFragment();
-        let pySrc = pycResult.toString();
+        if (!pySrc) {
+            let genStartTS = process.hrtime.bigint();
+            let decompiler = new PycDecompiler(obj);
+            let ast = decompiler.decompile();
+            let pycResult = ast.codeFragment();
+            pySrc = pycResult.toString();
+            genSecs = Number(process.hrtime.bigint() - genStartTS) / 1000000000;
+        }
         if (!pySrc.endsWith("\n")) {
             pySrc += "\n";
         }
-        let genSecs = parseInt(process.hrtime.bigint() - genStartTS) / 1000000000;
+        genSecs = Math.max(genSecs, 0.000000001);
         if (g_cliArgs.sendToStdout) {
 //            console.log(`\n\n${filenameBase}.${g_cliArgs.fileExt}\n-------\n${pySrc}`);
             console.log(pySrc);
@@ -156,9 +308,9 @@ function decompilePycObject(data) {
         }
         let secs = parseInt(process.hrtime.bigint() - startTS) / 1000000000;
         g_totalExecTime += secs;
-        let inThroughput = data.length / genSecs;
+        let inThroughput = buffer.length / genSecs;
         let outThroughput = pySrc.length / genSecs;
-        g_totalInThroughput += data.length;
+        g_totalInThroughput += buffer.length;
         g_totalOutThroughput += pySrc.length;
         g_totalFiles++;
         if (g_cliArgs.stats) {
@@ -196,6 +348,17 @@ function DecompileModule(filenames)
 }
 
 parseCLIParams()
+if (g_cliArgs.pyVersion && !g_cliArgs.marshal) {
+    console.log("Error: --py-version requires --marshal (headerless input).");
+    process.exit(1);
+}
+if (g_cliArgs.pyVersion) {
+    g_pyVersionInfo = PycReader.ResolveVersionTag(g_cliArgs.pyVersion);
+    if (!g_pyVersionInfo) {
+        console.log(`Error: unsupported --py-version "${g_cliArgs.pyVersion}". Use format X.Y (e.g., 3.11).`);
+        process.exit(1);
+    }
+}
 if (g_cliArgs.filenames.length === 0) {
     printUsage();
     process.exit(1);

package/lib/PycDecompiler.js

@@ -712,7 +712,9 @@ class PycDecompiler {
                 {
                     PycDecompiler.opCodeHandlers[this.code.Current.OpCodeID].call(this);
                 } else {
-                    console.error(`Unsupported opcode ${this.code.Current.InstructionName} at pos ${this.code.Current.Offset}\n`);
+                    if (!g_cliArgs?.silent) {
+                        console.error(`Unsupported opcode ${this.code.Current.InstructionName} at pos ${this.code.Current.Offset}\n`);
+                    }
                     this.cleanBuild = false;
                     let node = new AST.ASTNodeList(this.defBlock.nodes);
                     return node;
@@ -725,9 +727,11 @@ class PycDecompiler {
                         && (this.curBlock.end == this.code.Next?.Offset);
 
             } catch (ex) {
-                console.error(`EXCEPTION for OpCode ${this.code.Current.InstructionName} (${this.code.Current.Argument}) at offset ${this.code.Current.Offset} in code object '${this.object.Name}', file offset ${this.object.codeOffset + this.code.Current.Offset} : ${ex.message}\n\n`);
-                if (global.g_cliArgs?.debug) {
-                    console.error('Stack trace:', ex.stack);
+                if (!g_cliArgs?.silent) {
+                    console.error(`EXCEPTION for OpCode ${this.code.Current.InstructionName} (${this.code.Current.Argument}) at offset ${this.code.Current.Offset} in code object '${this.object.Name}', file offset ${this.object.codeOffset + this.code.Current.Offset} : ${ex.message}\n\n`);
+                    if (global.g_cliArgs?.debug) {
+                        console.error('Stack trace:', ex.stack);
+                    }
                 }
             }
         }

package/lib/PycReader.js

@@ -103,6 +103,37 @@ const MagicToVersion = {
     0x0A0D0E2B: {major: 3, minor: 14, IsUnicode: true, opcode: require('./bytecode/python_3_14')}
 };
 
+const VersionAliases = {
+    "1.2": "1.1"
+};
+
+const VersionToInfo = {};
+for (const [magic, info] of Object.entries(MagicToVersion)) {
+    const key = `${info.major}.${info.minor}`;
+    const existing = VersionToInfo[key];
+    const candidate = {...info, magic: Number(magic)};
+    if (!existing || (candidate.revision || 0) > (existing.revision || 0)) {
+        VersionToInfo[key] = candidate;
+    }
+}
+
+function parseVersionTag(tag) {
+    if (!tag) {
+        return null;
+    }
+    const cleaned = String(tag).trim().replace(/^python/i, '').replace(/^py/i, '');
+    const match = cleaned.match(/^(\d+)(?:\.(\d+))?(?:\.(\d+))?$/);
+    if (!match || match[2] === undefined) {
+        return null;
+    }
+    const major = parseInt(match[1], 10);
+    const minor = parseInt(match[2], 10);
+    if (Number.isNaN(major) || Number.isNaN(minor)) {
+        return null;
+    }
+    return {major, minor};
+}
+
 
 class PycReader
 {
@@ -123,11 +154,14 @@ class PycReader
     m_filename = null;
     m_version = null;
 
-    constructor(data) {
+    constructor(data, options = {}) {
         if (!data) {
             return;
         }
         
+        const opts = options || {};
+        this.m_filename = opts.filename || null;
+
         let buffer = data;
         if (typeof(data) == 'string') {
             buffer = fs.readFileSync(data);
@@ -135,6 +169,21 @@ class PycReader
             throw new Error('PycReader accepts only String as a file path or Buffer as content.');
         }
         this.m_rdr = new BinaryReader(buffer);
+
+        if (opts.marshal) {
+            let versionInfo = opts.versionInfo;
+            if (!versionInfo && opts.pyVersion) {
+                versionInfo = PycReader.ResolveVersionTag(opts.pyVersion);
+            }
+            if (!versionInfo) {
+                throw new Error('Marshal mode requires a bytecode version. Pass --py-version or pre-resolve it.');
+            }
+            this.m_version = versionInfo;
+            if (global.g_cliArgs?.debug && !opts.silent) {
+                console.log(`Marshal mode: Python ${this.m_version.major}.${this.m_version.minor}`);
+            }
+            return;
+        }
         
         let marshalVersion = this.m_rdr.readUInt32();
         this.m_version = MagicToVersion[marshalVersion] || MagicToVersion[marshalVersion & 0xFFFFFFFE] || {major: -1, minor: -1, IsUnicode: false};
@@ -190,6 +239,129 @@ class PycReader
         return this.m_version.opcode;
     }
 
+    static ResolveVersionTag(tag) {
+        const parsed = parseVersionTag(tag);
+        if (!parsed) {
+            return null;
+        }
+        const key = `${parsed.major}.${parsed.minor}`;
+        const aliased = VersionAliases[key] ? VersionAliases[key] : key;
+        return VersionToInfo[aliased] || null;
+    }
+
+    static ListSupportedVersions(desc = true) {
+        const list = Object.values(VersionToInfo);
+        list.sort((a, b) => {
+            if (a.major !== b.major) {
+                return a.major - b.major;
+            }
+            return a.minor - b.minor;
+        });
+        return desc ? list.reverse() : list;
+    }
+
+    static GuessVersion(buffer) {
+        const candidates = PycReader.ListSupportedVersions(true);
+        let best = null;
+        for (const candidate of candidates) {
+            const trial = PycReader.TryParseMarshal(buffer, candidate);
+            if (!trial) {
+                continue;
+            }
+            if (trial.unknown === 0 && trial.remaining === 0) {
+                return candidate;
+            }
+            if (!best ||
+                trial.unknownRatio < best.unknownRatio ||
+                (trial.unknownRatio === best.unknownRatio && trial.remaining < best.remaining)) {
+                best = {...trial, candidate};
+            }
+        }
+        return best ? best.candidate : null;
+    }
+
+    static CountUnknownOpcodes(codeObject, reader, opCodeList) {
+        const code = codeObject?.Code?.Value;
+        if (!code || !opCodeList) {
+            return {unknown: Number.MAX_SAFE_INTEGER, total: 0};
+        }
+        const wordSize = reader.getInstructionWordSize();
+        let unknown = 0;
+        let total = 0;
+        if (wordSize === 2) {
+            for (let offset = 0; offset < code.length; offset += 2) {
+                total++;
+                if (!opCodeList[code[offset]]) {
+                    unknown++;
+                }
+            }
+        } else {
+            let offset = 0;
+            while (offset < code.length) {
+                total++;
+                const opcode = code[offset];
+                const entry = opCodeList[opcode];
+                if (!entry) {
+                    unknown++;
+                    offset += 1;
+                    continue;
+                }
+                offset += entry.HasArgument ? 3 : 1;
+            }
+        }
+        return {unknown, total};
+    }
+
+    static TryParseMarshal(buffer, versionInfo) {
+        try {
+            const reader = new PycReader(buffer, {marshal: true, versionInfo, silent: true});
+            const obj = reader.ReadObject();
+            if (!obj || obj.ClassName !== "Py_CodeObject") {
+                return null;
+            }
+            const remaining = reader.m_rdr.Reader.length - reader.m_rdr.pc;
+            if (remaining < 0) {
+                return null;
+            }
+
+            const prevDebug = global.g_cliArgs?.debug;
+            if (global.g_cliArgs) {
+                global.g_cliArgs.debug = false;
+            }
+            let opcodes = null;
+            try {
+                opcodes = new versionInfo.opcode(obj);
+            } finally {
+                if (global.g_cliArgs) {
+                    global.g_cliArgs.debug = prevDebug;
+                }
+            }
+
+            const {unknown, total} = PycReader.CountUnknownOpcodes(obj, reader, opcodes?.OpCodeList);
+            const unknownRatio = total > 0 ? unknown / total : 1;
+
+            let score = 0;
+            if (remaining === 0) {
+                score += 3;
+            }
+            if (unknown === 0) {
+                score += 2;
+            }
+            if (obj.Code?.Value && Buffer.isBuffer(obj.Code.Value)) {
+                score += 1;
+            }
+            if (obj.Consts?.ClassName === "Py_Tuple") {
+                score += 1;
+            }
+            if (obj.Names?.ClassName === "Py_Tuple") {
+                score += 1;
+            }
+            return {score, remaining, unknown, unknownRatio};
+        } catch (ex) {
+            return null;
+        }
+    }
+
     ReadObject() {
         try {
             let obj = new PythonObject(), value = null;

package/lib/ast/ast_node.js

@@ -1863,7 +1863,7 @@ class ASTKwNamesMap extends ASTNode {
 
     constructor(values) {
         super();
-        this.m_values = values;
+        this.m_values = Array.isArray(values) ? values : [];
     }
 
     get values() {
@@ -1876,6 +1876,9 @@ class ASTKwNamesMap extends ASTNode {
     }
 
     add (key, value) {
+        if (!this.m_values) {
+            this.m_values = [];
+        }
         this.values.push({key, value});
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "depyo",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Python bytecode decompiler (Python 1.0–3.14) implemented in Node.js",
   "bin": {
     "depyo": "./depyo.js"