@generaltranslation/python-extractor 0.0.0

package/dist/resolveFunctionVariants.js

@@ -0,0 +1,122 @@
+import fs from 'node:fs';
+import { getParser } from './parser.js';
+const crossFileCache = new Map();
+/**
+ * Resolves all return values of a function defined in the current file's AST.
+ * Uses the provided expression parser to handle complex return expressions
+ * (concat, declare_var, etc.).
+ */
+export async function resolveFunctionInCurrentFile(functionName, rootNode, parseExpr) {
+    const funcDef = findFunctionDefinition(rootNode, functionName);
+    if (!funcDef)
+        return null;
+    return extractReturnVariants(funcDef, rootNode, parseExpr);
+}
+/**
+ * Resolves all return values of a function defined in an external file.
+ * Results are cached by filePath::functionName.
+ */
+export async function resolveFunctionInFile(functionName, filePath, parseExpr) {
+    const cacheKey = `${filePath}::${functionName}`;
+    if (crossFileCache.has(cacheKey)) {
+        return crossFileCache.get(cacheKey);
+    }
+    let source;
+    try {
+        source = fs.readFileSync(filePath, 'utf8');
+    }
+    catch {
+        crossFileCache.set(cacheKey, null);
+        return null;
+    }
+    const parser = await getParser();
+    const tree = parser.parse(source);
+    if (!tree) {
+        crossFileCache.set(cacheKey, null);
+        return null;
+    }
+    const result = await resolveFunctionInCurrentFile(functionName, tree.rootNode, parseExpr);
+    crossFileCache.set(cacheKey, result);
+    return result;
+}
+/**
+ * Finds a top-level function_definition by name in the AST.
+ */
+function findFunctionDefinition(rootNode, name) {
+    for (let i = 0; i < rootNode.childCount; i++) {
+        const child = rootNode.child(i);
+        if (!child)
+            continue;
+        if (child.type === 'function_definition') {
+            const nameNode = child.childForFieldName('name');
+            if (nameNode && nameNode.text === name) {
+                return child;
+            }
+        }
+        // Also check decorated definitions
+        if (child.type === 'decorated_definition') {
+            const defNode = child.childForFieldName('definition');
+            if (defNode && defNode.type === 'function_definition') {
+                const nameNode = defNode.childForFieldName('name');
+                if (nameNode && nameNode.text === name) {
+                    return defNode;
+                }
+            }
+        }
+    }
+    return null;
+}
+/**
+ * Extracts all return values from a function body and parses them into StringNodes.
+ * Skips nested function definitions.
+ */
+async function extractReturnVariants(funcDef, rootNode, parseExpr) {
+    const body = funcDef.childForFieldName('body');
+    if (!body)
+        return null;
+    const returnExprs = [];
+    collectReturnExpressions(body, returnExprs);
+    if (returnExprs.length === 0)
+        return null;
+    // Parse each return expression into a StringNode
+    const nodes = [];
+    for (const expr of returnExprs) {
+        const node = await parseExpr(expr, rootNode);
+        if (node)
+            nodes.push(node);
+    }
+    if (nodes.length === 0)
+        return null;
+    if (nodes.length === 1)
+        return nodes[0];
+    return { type: 'choice', nodes };
+}
+/**
+ * Recursively collects return expression nodes from a function body,
+ * skipping nested function definitions.
+ */
+function collectReturnExpressions(node, results) {
+    if (node.type === 'function_definition') {
+        // Skip nested function bodies
+        return;
+    }
+    if (node.type === 'return_statement') {
+        // Get the expression being returned (skip the 'return' keyword)
+        for (let i = 0; i < node.childCount; i++) {
+            const child = node.child(i);
+            if (!child || child.type === 'return')
+                continue;
+            results.push(child);
+            break; // Only one expression per return
+        }
+        return;
+    }
+    for (let i = 0; i < node.childCount; i++) {
+        const child = node.child(i);
+        if (child)
+            collectReturnExpressions(child, results);
+    }
+}
+export function clearFunctionCache() {
+    crossFileCache.clear();
+}

package/dist/resolveImport.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Resolves a Python import module path to a file path on disk.
+ *
+ * Handles:
+ * - Relative imports: `.helpers` → sibling `helpers.py`
+ * - Parent relative imports: `..utils` → parent dir `utils.py`
+ * - Absolute imports: `utils` → `utils.py` in same/parent dirs
+ * - Dotted paths: `myapp.utils` → `myapp/utils.py`
+ */
+export declare function resolveImportPath(moduleName: string, currentFilePath: string): string | null;
+export declare function clearResolveCache(): void;

package/dist/resolveImport.js

@@ -0,0 +1,63 @@
+import fs from 'node:fs';
+import path from 'node:path';
+const resolveCache = new Map();
+/**
+ * Resolves a Python import module path to a file path on disk.
+ *
+ * Handles:
+ * - Relative imports: `.helpers` → sibling `helpers.py`
+ * - Parent relative imports: `..utils` → parent dir `utils.py`
+ * - Absolute imports: `utils` → `utils.py` in same/parent dirs
+ * - Dotted paths: `myapp.utils` → `myapp/utils.py`
+ */
+export function resolveImportPath(moduleName, currentFilePath) {
+    const cacheKey = `${currentFilePath}::${moduleName}`;
+    if (resolveCache.has(cacheKey)) {
+        return resolveCache.get(cacheKey);
+    }
+    const result = doResolve(moduleName, currentFilePath);
+    resolveCache.set(cacheKey, result);
+    return result;
+}
+function doResolve(moduleName, currentFilePath) {
+    const currentDir = path.dirname(currentFilePath);
+    // Relative import: starts with dots
+    if (moduleName.startsWith('.')) {
+        // Count leading dots
+        let dotCount = 0;
+        while (dotCount < moduleName.length && moduleName[dotCount] === '.') {
+            dotCount++;
+        }
+        // Go up (dotCount - 1) directories from current dir
+        let baseDir = currentDir;
+        for (let i = 1; i < dotCount; i++) {
+            baseDir = path.dirname(baseDir);
+        }
+        const remainder = moduleName.slice(dotCount);
+        if (!remainder)
+            return null;
+        return resolveModulePath(baseDir, remainder);
+    }
+    // Absolute import: dotted or simple name
+    return resolveModulePath(currentDir, moduleName);
+}
+/**
+ * Resolves a dotted module name (e.g. "myapp.utils") relative to a base dir.
+ * Tries: baseDir/myapp/utils.py, then baseDir/myapp/utils/__init__.py
+ */
+function resolveModulePath(baseDir, moduleName) {
+    const parts = moduleName.split('.');
+    const filePath = path.join(baseDir, ...parts) + '.py';
+    if (fs.existsSync(filePath)) {
+        return filePath;
+    }
+    // Try as package: moduleName/__init__.py
+    const initPath = path.join(baseDir, ...parts, '__init__.py');
+    if (fs.existsSync(initPath)) {
+        return initPath;
+    }
+    return null;
+}
+export function clearResolveCache() {
+    resolveCache.clear();
+}

package/dist/stringNode.d.ts

@@ -0,0 +1,21 @@
+type StringNode = StringTextNode | StringSequenceNode | StringChoiceNode;
+type StringTextNode = {
+    type: 'text';
+    text: string;
+};
+type StringSequenceNode = {
+    type: 'sequence';
+    nodes: StringNode[];
+};
+type StringChoiceNode = {
+    type: 'choice';
+    nodes: StringNode[];
+};
+export type { StringNode, StringTextNode, StringSequenceNode, StringChoiceNode, };
+/**
+ * Converts a StringNode tree into all possible string variants.
+ * - TextNode → single string
+ * - SequenceNode → cartesian product of all parts
+ * - ChoiceNode → flattened branches (deduplicated)
+ */
+export declare function nodeToStrings(node: StringNode | null): string[];

package/dist/stringNode.js

@@ -0,0 +1,51 @@
+// ===== Tree Construction ===== //
+// Used for declare_static / declare_var parsing
+/**
+ * Converts a StringNode tree into all possible string variants.
+ * - TextNode → single string
+ * - SequenceNode → cartesian product of all parts
+ * - ChoiceNode → flattened branches (deduplicated)
+ */
+export function nodeToStrings(node) {
+    if (node === null) {
+        return [];
+    }
+    if (node.type === 'text') {
+        return [node.text];
+    }
+    if (node.type === 'sequence') {
+        const partResults = node.nodes.map((n) => nodeToStrings(n));
+        return cartesianProduct(partResults);
+    }
+    if (node.type === 'choice') {
+        const allStrings = [];
+        for (const branch of node.nodes) {
+            allStrings.push(...nodeToStrings(branch));
+        }
+        return [...new Set(allStrings)]; // Deduplicate
+    }
+    return [];
+}
+/**
+ * Creates cartesian product of string arrays and concatenates them.
+ * @example cartesianProduct([["Hello "], ["day", "night"]]) → ["Hello day", "Hello night"]
+ */
+function cartesianProduct(arrays) {
+    if (arrays.length === 0) {
+        return [];
+    }
+    if (arrays.length === 1) {
+        return arrays[0];
+    }
+    let result = arrays[0];
+    for (let i = 1; i < arrays.length; i++) {
+        const newResult = [];
+        for (const prev of result) {
+            for (const curr of arrays[i]) {
+                newResult.push(prev + curr);
+            }
+        }
+        result = newResult;
+    }
+    return result;
+}

package/dist/types.d.ts

@@ -0,0 +1,16 @@
+export type ExtractionResult = {
+    /** The data format of the extracted content */
+    dataFormat: 'ICU' | 'JSX';
+    /** The extracted translatable content */
+    source: string;
+    /** Metadata about the extraction */
+    metadata: ExtractionMetadata;
+};
+export type ExtractionMetadata = {
+    id?: string;
+    context?: string;
+    maxChars?: number;
+    filePaths?: string[];
+    /** Groups related static content variants together (for declareStatic/declare_static) */
+    staticId?: string;
+};

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@generaltranslation/python-extractor",
+  "version": "0.0.0",
+  "description": "Python source code extraction for General Translation",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "type": "module",
+  "author": "General Translation, Inc.",
+  "license": "FSL-1.1-ALv2",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./types": {
+      "import": "./dist/types.js",
+      "types": "./dist/types.d.ts"
+    },
+    "./constants": {
+      "import": "./dist/constants.js",
+      "types": "./dist/constants.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/generaltranslation/gt.git"
+  },
+  "keywords": [
+    "python",
+    "extraction",
+    "translation",
+    "i18n",
+    "flask",
+    "fastapi"
+  ],
+  "devDependencies": {
+    "@types/node": "^22.5.1",
+    "typescript": "^5.5.4",
+    "vitest": "^2.0.0"
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "dependencies": {
+    "tree-sitter-python": "^0.25.0",
+    "web-tree-sitter": "^0.26.6",
+    "generaltranslation": "8.1.14"
+  },
+  "scripts": {
+    "build": "tsc",
+    "build:watch": "tsc --watch",
+    "build:clean": "sh ../../scripts/clean.sh && pnpm run build",
+    "build:release": "pnpm run build:clean",
+    "format": "prettier --write src",
+    "patch": "pnpm version patch",
+    "release": "pnpm run build:clean && pnpm publish",
+    "release:alpha": "pnpm run build:clean && pnpm publish --tag alpha",
+    "release:beta": "pnpm run build:clean && pnpm publish --tag beta",
+    "release:latest": "pnpm run build:clean && pnpm publish --tag latest",
+    "lint": "eslint \"src/**/*.{js,ts}\"",
+    "lint:fix": "eslint \"src/**/*.{js,ts}\" --fix",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}