@kernlang/review-python 0.2.0

package/dist/mapper.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Python Concept Mapper — tree-sitter based.
+ *
+ * Maps Python syntax → universal KERN concepts.
+ * Phase 1: error_raise, error_handle, effect
+ */
+import type { ConceptMap } from '@kernlang/core';
+export declare function extractPythonConcepts(source: string, filePath: string): ConceptMap;

package/dist/mapper.js

@@ -0,0 +1,522 @@
+/**
+ * Python Concept Mapper — tree-sitter based.
+ *
+ * Maps Python syntax → universal KERN concepts.
+ * Phase 1: error_raise, error_handle, effect
+ */
+import Parser from 'tree-sitter';
+import Python from 'tree-sitter-python';
+import { conceptId, conceptSpan } from '@kernlang/core';
+const EXTRACTOR_VERSION = '1.0.0';
+// ── Network call patterns ────────────────────────────────────────────────
+const NETWORK_MODULES = new Set(['requests', 'httpx', 'aiohttp', 'urllib']);
+const NETWORK_METHODS = new Set(['get', 'post', 'put', 'patch', 'delete', 'head', 'options', 'request', 'fetch']);
+const DB_MODULES = new Set(['psycopg2', 'asyncpg', 'pymongo', 'sqlalchemy', 'django']);
+const DB_METHODS = new Set(['execute', 'executemany', 'fetchone', 'fetchall', 'fetchmany', 'query', 'find', 'find_one', 'insert_one', 'insert_many', 'update_one', 'delete_one']);
+const FS_FUNCTIONS = new Set(['open', 'read', 'write', 'readlines', 'writelines']);
+const STDLIB_MODULES = new Set([
+    'os', 'sys', 'json', 're', 'math', 'datetime', 'time', 'logging', 'argparse',
+    'collections', 'itertools', 'functools', 'pathlib', 'shutil', 'subprocess',
+    'threading', 'multiprocessing', 'abc', 'typing', 'io', 'pickle', 'random',
+    'hashlib', 'hmac', 'base64', 'csv', 'sqlite3', 'zlib', 'gzip', 'tarfile', 'zipfile',
+    'enum', 'struct', 'tempfile', 'unittest', 'urllib', 'uuid', 'xml',
+]);
+// ── Parser setup ─────────────────────────────────────────────────────────
+let parser = null;
+function getParser() {
+    if (!parser) {
+        parser = new Parser();
+        parser.setLanguage(Python);
+    }
+    return parser;
+}
+// ── Main Extractor ───────────────────────────────────────────────────────
+export function extractPythonConcepts(source, filePath) {
+    const tree = getParser().parse(source);
+    const nodes = [];
+    const edges = [];
+    extractErrorRaise(tree.rootNode, source, filePath, nodes);
+    extractErrorHandle(tree.rootNode, source, filePath, nodes);
+    extractEffects(tree.rootNode, source, filePath, nodes);
+    extractEntrypoints(tree.rootNode, source, filePath, nodes);
+    extractGuards(tree.rootNode, source, filePath, nodes);
+    extractStateMutation(tree.rootNode, source, filePath, nodes);
+    extractDependencyEdges(tree.rootNode, source, filePath, edges);
+    return {
+        filePath,
+        language: 'py',
+        nodes,
+        edges,
+        extractorVersion: EXTRACTOR_VERSION,
+    };
+}
+// ── error_raise ──────────────────────────────────────────────────────────
+function extractErrorRaise(root, source, filePath, nodes) {
+    // raise statements
+    walkNodes(root, 'raise_statement', (node) => {
+        const errorType = extractRaiseType(node);
+        nodes.push({
+            id: conceptId(filePath, 'error_raise', node.startIndex),
+            kind: 'error_raise',
+            primarySpan: nodeSpan(filePath, node),
+            evidence: nodeText(source, node, 100),
+            confidence: 1.0,
+            language: 'py',
+            containerId: getContainerId(node, filePath),
+            payload: {
+                kind: 'error_raise',
+                subtype: 'throw', // Python raise ≡ throw
+                errorType,
+            },
+        });
+    });
+}
+// ── error_handle ─────────────────────────────────────────────────────────
+function extractErrorHandle(root, source, filePath, nodes) {
+    // except clauses
+    walkNodes(root, 'except_clause', (node) => {
+        const block = node.children.find(c => c.type === 'block');
+        const disposition = classifyPythonDisposition(block, source);
+        const errorVar = extractExceptVar(node);
+        nodes.push({
+            id: conceptId(filePath, 'error_handle', node.startIndex),
+            kind: 'error_handle',
+            primarySpan: nodeSpan(filePath, node),
+            evidence: nodeText(source, node, 150),
+            confidence: disposition.confidence,
+            language: 'py',
+            containerId: getContainerId(node, filePath),
+            payload: {
+                kind: 'error_handle',
+                disposition: disposition.type,
+                errorVariable: errorVar,
+            },
+        });
+    });
+}
+function classifyPythonDisposition(block, source) {
+    if (!block)
+        return { type: 'ignored', confidence: 1.0 };
+    const children = block.namedChildren;
+    // except: pass → ignored
+    if (children.length === 1 && children[0].type === 'pass_statement') {
+        return { type: 'ignored', confidence: 1.0 };
+    }
+    // except: ... (ellipsis) → ignored
+    if (children.length === 1 && children[0].type === 'expression_statement') {
+        const text = source.substring(children[0].startIndex, children[0].endIndex).trim();
+        if (text === '...')
+            return { type: 'ignored', confidence: 1.0 };
+    }
+    // Empty block
+    if (children.length === 0) {
+        return { type: 'ignored', confidence: 1.0 };
+    }
+    const bodyText = source.substring(block.startIndex, block.endIndex);
+    // raise → rethrown or wrapped
+    if (bodyText.includes('raise')) {
+        // bare `raise` → rethrown
+        if (/\braise\s*$|\braise\s*\n/m.test(bodyText)) {
+            return { type: 'rethrown', confidence: 0.95 };
+        }
+        return { type: 'wrapped', confidence: 0.9 };
+    }
+    // return → returned
+    if (bodyText.includes('return')) {
+        return { type: 'returned', confidence: 0.85 };
+    }
+    // logging
+    if (/\b(logging|logger|log|print)\b/.test(bodyText)) {
+        if (children.length === 1)
+            return { type: 'logged', confidence: 0.9 };
+        return { type: 'logged', confidence: 0.7 };
+    }
+    return { type: 'wrapped', confidence: 0.5 };
+}
+// ── effect ───────────────────────────────────────────────────────────────
+function extractEffects(root, source, filePath, nodes) {
+    walkNodes(root, 'call', (node) => {
+        const funcNode = node.childForFieldName('function');
+        if (!funcNode)
+            return;
+        const funcText = source.substring(funcNode.startIndex, funcNode.endIndex);
+        // Network: requests.get(), httpx.post(), etc.
+        if (funcNode.type === 'attribute') {
+            const obj = funcNode.childForFieldName('object');
+            const attr = funcNode.childForFieldName('attribute');
+            if (obj && attr) {
+                const objName = source.substring(obj.startIndex, obj.endIndex);
+                const methodName = source.substring(attr.startIndex, attr.endIndex);
+                if (NETWORK_MODULES.has(objName) && NETWORK_METHODS.has(methodName)) {
+                    nodes.push({
+                        id: conceptId(filePath, 'effect', node.startIndex),
+                        kind: 'effect',
+                        primarySpan: nodeSpan(filePath, node),
+                        evidence: nodeText(source, node, 120),
+                        confidence: 0.95,
+                        language: 'py',
+                        containerId: getContainerId(node, filePath),
+                        payload: { kind: 'effect', subtype: 'network', async: isInAsyncDef(node) },
+                    });
+                    return;
+                }
+                // DB: cursor.execute(), db.query(), etc.
+                if (DB_METHODS.has(methodName) && (DB_MODULES.has(objName) || /cursor|conn|db|session|collection/i.test(objName))) {
+                    nodes.push({
+                        id: conceptId(filePath, 'effect', node.startIndex),
+                        kind: 'effect',
+                        primarySpan: nodeSpan(filePath, node),
+                        evidence: nodeText(source, node, 120),
+                        confidence: 0.85,
+                        language: 'py',
+                        containerId: getContainerId(node, filePath),
+                        payload: { kind: 'effect', subtype: 'db', async: isInAsyncDef(node) },
+                    });
+                    return;
+                }
+            }
+        }
+        // FS: open()
+        if (funcText === 'open') {
+            nodes.push({
+                id: conceptId(filePath, 'effect', node.startIndex),
+                kind: 'effect',
+                primarySpan: nodeSpan(filePath, node),
+                evidence: nodeText(source, node, 120),
+                confidence: 0.9,
+                language: 'py',
+                containerId: getContainerId(node, filePath),
+                payload: { kind: 'effect', subtype: 'fs', async: false },
+            });
+        }
+        // fetch() in async context (aiohttp pattern)
+        if (funcText === 'fetch' || funcText === 'aiohttp.request') {
+            nodes.push({
+                id: conceptId(filePath, 'effect', node.startIndex),
+                kind: 'effect',
+                primarySpan: nodeSpan(filePath, node),
+                evidence: nodeText(source, node, 120),
+                confidence: 0.8,
+                language: 'py',
+                containerId: getContainerId(node, filePath),
+                payload: { kind: 'effect', subtype: 'network', async: true },
+            });
+        }
+    });
+}
+// ── entrypoint ──────────────────────────────────────────────────────────
+function extractEntrypoints(root, source, filePath, nodes) {
+    // 1. Route decorators: @app.route, @app.get, @router.post, etc.
+    // tree-sitter Python wraps decorated functions in 'decorated_definition'
+    walkNodes(root, 'decorated_definition', (node) => {
+        const fnDef = node.children.find(c => c.type === 'function_definition');
+        if (!fnDef)
+            return;
+        for (const child of node.children) {
+            if (child.type !== 'decorator')
+                continue;
+            const decText = source.substring(child.startIndex, child.endIndex);
+            const routeMatch = decText.match(/@(app|router|bp)\.(route|get|post|put|delete|patch)\s*\(/);
+            if (routeMatch) {
+                const method = routeMatch[2].toUpperCase();
+                const nameNode = fnDef.childForFieldName('name');
+                // Try to extract path from decorator args
+                const pathMatch = decText.match(/['"]([^'"]+)['"]/);
+                nodes.push({
+                    id: conceptId(filePath, 'entrypoint', child.startIndex),
+                    kind: 'entrypoint',
+                    primarySpan: nodeSpan(filePath, child),
+                    evidence: nodeText(source, child, 100),
+                    confidence: 1.0,
+                    language: 'py',
+                    containerId: getContainerId(node, filePath),
+                    payload: {
+                        kind: 'entrypoint',
+                        subtype: 'route',
+                        name: nameNode ? nameNode.text : (pathMatch?.[1] || 'anonymous'),
+                        httpMethod: method === 'ROUTE' ? undefined : method,
+                    },
+                });
+            }
+        }
+    });
+    // 2. if __name__ == '__main__':
+    walkNodes(root, 'if_statement', (node) => {
+        const condition = node.childForFieldName('condition');
+        if (condition && condition.text.includes('__name__') && condition.text.includes('__main__')) {
+            nodes.push({
+                id: conceptId(filePath, 'entrypoint', node.startIndex),
+                kind: 'entrypoint',
+                primarySpan: nodeSpan(filePath, node),
+                evidence: nodeText(source, node, 100),
+                confidence: 1.0,
+                language: 'py',
+                payload: {
+                    kind: 'entrypoint',
+                    subtype: 'main',
+                    name: 'main',
+                },
+            });
+        }
+    });
+}
+// ── guard ───────────────────────────────────────────────────────────────
+function extractGuards(root, source, filePath, nodes) {
+    // 1. Auth decorators (tree-sitter: decorated_definition → decorator + function_definition)
+    walkNodes(root, 'decorated_definition', (node) => {
+        for (const child of node.children) {
+            if (child.type !== 'decorator')
+                continue;
+            const decText = source.substring(child.startIndex, child.endIndex);
+            if (/@(login_required|requires_auth|permission_required|auth_required|authenticated)/.test(decText)) {
+                nodes.push({
+                    id: conceptId(filePath, 'guard', child.startIndex),
+                    kind: 'guard',
+                    primarySpan: nodeSpan(filePath, child),
+                    evidence: nodeText(source, child, 100),
+                    confidence: 1.0,
+                    language: 'py',
+                    containerId: getContainerId(node, filePath),
+                    payload: {
+                        kind: 'guard',
+                        subtype: 'auth',
+                        name: decText.replace('@', '').split('(')[0].trim(),
+                    },
+                });
+            }
+        }
+    });
+    // 2. Pydantic validation: BaseModel.model_validate()
+    walkNodes(root, 'call', (node) => {
+        const func = node.childForFieldName('function');
+        if (func && func.text.includes('model_validate')) {
+            nodes.push({
+                id: conceptId(filePath, 'guard', node.startIndex),
+                kind: 'guard',
+                primarySpan: nodeSpan(filePath, node),
+                evidence: nodeText(source, node, 100),
+                confidence: 0.9,
+                language: 'py',
+                containerId: getContainerId(node, filePath),
+                payload: { kind: 'guard', subtype: 'validation', name: 'pydantic' },
+            });
+        }
+    });
+    // 3. Early return/raise after auth check: if not request.user: raise/return
+    walkNodes(root, 'if_statement', (node) => {
+        const cond = node.childForFieldName('condition');
+        if (cond && /\b(user|auth|request\.user)\b/.test(cond.text)) {
+            const block = node.namedChildren.find(c => c.type === 'block');
+            if (block) {
+                const firstStmt = block.namedChildren[0];
+                if (firstStmt && (firstStmt.type === 'return_statement' || firstStmt.type === 'raise_statement')) {
+                    nodes.push({
+                        id: conceptId(filePath, 'guard', node.startIndex),
+                        kind: 'guard',
+                        primarySpan: nodeSpan(filePath, node),
+                        evidence: nodeText(source, node, 100),
+                        confidence: 0.8,
+                        language: 'py',
+                        containerId: getContainerId(node, filePath),
+                        payload: { kind: 'guard', subtype: 'auth' },
+                    });
+                }
+            }
+        }
+    });
+}
+// ── state_mutation ───────────────────────────────────────────────────────
+function extractStateMutation(root, source, filePath, nodes) {
+    // Track global keyword usage
+    const globalVarsInFile = new Set();
+    walkNodes(root, 'global_statement', (node) => {
+        for (const child of node.namedChildren) {
+            if (child.type === 'identifier')
+                globalVarsInFile.add(child.text);
+        }
+    });
+    walkNodes(root, 'assignment', (node) => {
+        const left = node.childForFieldName('left');
+        if (!left)
+            return;
+        // self.x = ... → scope 'module' (as requested)
+        if (left.type === 'attribute') {
+            const obj = left.childForFieldName('object');
+            if (obj && obj.text === 'self') {
+                nodes.push({
+                    id: conceptId(filePath, 'state_mutation', node.startIndex),
+                    kind: 'state_mutation',
+                    primarySpan: nodeSpan(filePath, node),
+                    evidence: nodeText(source, node, 100),
+                    confidence: 0.9,
+                    language: 'py',
+                    containerId: getContainerId(node, filePath),
+                    payload: { kind: 'state_mutation', target: left.text, scope: 'module' },
+                });
+                return;
+            }
+        }
+        // Global or Module level assignment
+        if (left.type === 'identifier') {
+            const name = left.text;
+            const containerId = getContainerId(node, filePath);
+            if (globalVarsInFile.has(name)) {
+                nodes.push({
+                    id: conceptId(filePath, 'state_mutation', node.startIndex),
+                    kind: 'state_mutation',
+                    primarySpan: nodeSpan(filePath, node),
+                    evidence: nodeText(source, node, 100),
+                    confidence: 1.0,
+                    language: 'py',
+                    containerId,
+                    payload: { kind: 'state_mutation', target: name, scope: 'global' },
+                });
+            }
+            else if (!containerId) {
+                // Module level (top level)
+                nodes.push({
+                    id: conceptId(filePath, 'state_mutation', node.startIndex),
+                    kind: 'state_mutation',
+                    primarySpan: nodeSpan(filePath, node),
+                    evidence: nodeText(source, node, 100),
+                    confidence: 0.8,
+                    language: 'py',
+                    payload: { kind: 'state_mutation', target: name, scope: 'module' },
+                });
+            }
+        }
+    });
+}
+// ── dependency ──────────────────────────────────────────────────────────
+function extractDependencyEdges(root, source, filePath, edges) {
+    const addDependency = (node, specifier) => {
+        let subtype = 'external';
+        if (specifier.startsWith('.')) {
+            subtype = 'internal';
+        }
+        else {
+            const rootModule = specifier.split('.')[0];
+            if (STDLIB_MODULES.has(rootModule)) {
+                subtype = 'stdlib';
+            }
+        }
+        edges.push({
+            id: `${filePath}#dep@${node.startIndex}`,
+            kind: 'dependency',
+            sourceId: filePath,
+            targetId: specifier,
+            primarySpan: nodeSpan(filePath, node),
+            evidence: nodeText(source, node, 100),
+            confidence: 1.0,
+            language: 'py',
+            payload: { kind: 'dependency', subtype, specifier },
+        });
+    };
+    walkNodes(root, 'import_statement', (node) => {
+        // import x, y as z
+        for (const child of node.namedChildren) {
+            if (child.type === 'dotted_name') {
+                addDependency(node, child.text);
+            }
+            else if (child.type === 'aliased_import') {
+                const name = child.childForFieldName('name');
+                if (name)
+                    addDependency(node, name.text);
+            }
+        }
+    });
+    walkNodes(root, 'import_from_statement', (node) => {
+        // from x import y
+        const moduleNode = node.childForFieldName('module_name');
+        const relativeMatch = node.text.match(/^from\s+(\.+)/);
+        let specifier = moduleNode ? moduleNode.text : '';
+        if (relativeMatch) {
+            specifier = relativeMatch[1] + specifier;
+        }
+        if (specifier) {
+            addDependency(node, specifier);
+        }
+    });
+}
+// ── Tree-sitter Helpers ──────────────────────────────────────────────────
+function walkNodes(root, type, callback) {
+    const cursor = root.walk();
+    let reachedRoot = false;
+    while (true) {
+        if (cursor.nodeType === type) {
+            callback(cursor.currentNode);
+        }
+        if (cursor.gotoFirstChild())
+            continue;
+        if (cursor.gotoNextSibling())
+            continue;
+        while (true) {
+            if (!cursor.gotoParent()) {
+                reachedRoot = true;
+                break;
+            }
+            if (cursor.gotoNextSibling())
+                break;
+        }
+        if (reachedRoot)
+            break;
+    }
+}
+function nodeSpan(filePath, node) {
+    return conceptSpan(filePath, node.startPosition.row + 1, node.startPosition.column + 1, node.endPosition.row + 1, node.endPosition.column + 1);
+}
+function nodeText(source, node, maxLen) {
+    return source.substring(node.startIndex, Math.min(node.endIndex, node.startIndex + maxLen));
+}
+function getContainerId(node, filePath) {
+    let parent = node.parent;
+    while (parent) {
+        if (parent.type === 'function_definition' || parent.type === 'class_definition') {
+            const nameNode = parent.childForFieldName('name');
+            const name = nameNode ? nameNode.text : 'anonymous';
+            return `${filePath}#fn:${name}@${parent.startIndex}`;
+        }
+        parent = parent.parent;
+    }
+    return undefined;
+}
+function extractRaiseType(node) {
+    // raise ValueError("...") → "ValueError"
+    const callNode = node.namedChildren.find(c => c.type === 'call');
+    if (callNode) {
+        const func = callNode.childForFieldName('function');
+        if (func)
+            return func.text;
+    }
+    // raise ValueError → just identifier
+    const ident = node.namedChildren.find(c => c.type === 'identifier');
+    if (ident)
+        return ident.text;
+    return undefined;
+}
+function extractExceptVar(node) {
+    // except Exception as e → "e"
+    for (const child of node.children) {
+        if (child.type === 'as_pattern') {
+            const alias = child.childForFieldName('alias');
+            if (alias)
+                return alias.text;
+        }
+        // Also try direct identifier after 'as'
+        if (child.type === 'identifier' && child.previousSibling?.text === 'as') {
+            return child.text;
+        }
+    }
+    return undefined;
+}
+function isInAsyncDef(node) {
+    let parent = node.parent;
+    while (parent) {
+        if (parent.type === 'function_definition') {
+            // Check for 'async' keyword before 'def'
+            return parent.children.some(c => c.type === 'async');
+        }
+        parent = parent.parent;
+    }
+    return false;
+}

package/jest.config.js

@@ -0,0 +1,10 @@
+export default {
+  testMatch: ['**/tests/**/*.test.ts'],
+  extensionsToTreatAsEsm: ['.ts'],
+  transform: {
+    '^.+\\.ts$': ['ts-jest', { useESM: true, tsconfig: 'tsconfig.json' }],
+  },
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+};

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@kernlang/review-python",
+  "version": "0.2.0",
+  "description": "Python concept mapper for kern review — tree-sitter based",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "tree-sitter": "^0.22.4",
+    "tree-sitter-python": "^0.23.6",
+    "@kernlang/core": "3.0.0",
+    "@kernlang/review": "3.0.0"
+  },
+  "devDependencies": {
+    "ts-morph": "^24.0.0",
+    "typescript": "^5.7.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "test": "node --experimental-vm-modules ../../node_modules/jest/bin/jest.js --forceExit --config jest.config.js"
+  }
+}

package/src/index.ts

@@ -0,0 +1,8 @@
+/**
+ * @kernlang/review-python — Python concept mapper using tree-sitter.
+ *
+ * Translates Python syntax into universal KERN concepts.
+ * Same ConceptMap output as the TypeScript mapper.
+ */
+
+export { extractPythonConcepts } from './mapper.js';