@generaltranslation/python-extractor 0.0.0

package/LICENSE.md

@@ -0,0 +1,105 @@
+# Functional Source License, Version 1.1, ALv2 Future License
+
+## Abbreviation
+
+FSL-1.1-ALv2
+
+## Notice
+
+Copyright 2025 General Translation, Inc.
+
+## Terms and Conditions
+
+### Licensor ("We")
+
+The party offering the Software under these Terms and Conditions.
+
+### The Software
+
+The "Software" is each version of the software that we make available under
+these Terms and Conditions, as indicated by our inclusion of these Terms and
+Conditions with the Software.
+
+### License Grant
+
+Subject to your compliance with this License Grant and the Patents,
+Redistribution and Trademark clauses below, we hereby grant you the right to
+use, copy, modify, create derivative works, publicly perform, publicly display
+and redistribute the Software for any Permitted Purpose identified below.
+
+### Permitted Purpose
+
+A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
+means making the Software available to others in a commercial product or
+service that:
+
+1. substitutes for the Software;
+
+2. substitutes for any other product or service we offer using the Software
+   that exists as of the date we make the Software available; or
+
+3. offers the same or substantially similar functionality as the Software.
+
+Permitted Purposes specifically include using the Software:
+
+1. for your internal use and access;
+
+2. for non-commercial education;
+
+3. for non-commercial research; and
+
+4. in connection with professional services that you provide to a licensee
+   using the Software in accordance with these Terms and Conditions.
+
+### Patents
+
+To the extent your use for a Permitted Purpose would necessarily infringe our
+patents, the license grant above includes a license under our patents. If you
+make a claim against any party that the Software infringes or contributes to
+the infringement of any patent, then your patent license to the Software ends
+immediately.
+
+### Redistribution
+
+The Terms and Conditions apply to all copies, modifications and derivatives of
+the Software.
+
+If you redistribute any copies, modifications or derivatives of the Software,
+you must include a copy of or a link to these Terms and Conditions and not
+remove any copyright notices provided in or with the Software.
+
+### Disclaimer
+
+THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
+PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
+
+IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
+SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
+EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
+
+### Trademarks
+
+Except for displaying the License Details and identifying us as the origin of
+the Software, you have no right under these Terms and Conditions to use our
+trademarks, trade names, service marks or product names.
+
+## Grant of Future License
+
+We hereby irrevocably grant you an additional license to use the Software under
+the Apache License, Version 2.0 that is effective on the second anniversary of
+the date we make the Software available. On or after that date, you may use the
+Software under the Apache License, Version 2.0, in which case the following
+will apply:
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.

package/dist/constants.d.ts

@@ -0,0 +1,13 @@
+export declare const PYTHON_GT_PACKAGES: readonly ["gt_flask", "gt_fastapi"];
+export declare const PYTHON_GT_DEPENDENCIES: readonly ["gt-flask", "gt-fastapi"];
+export declare const PYTHON_T_FUNCTION = "t";
+export declare const PYTHON_MSG_FUNCTION = "msg";
+export declare const PYTHON_DECLARE_STATIC = "declare_static";
+export declare const PYTHON_DECLARE_VAR = "declare_var";
+/** These imported names are tracked (translation functions + static helpers) */
+export declare const PYTHON_TRANSLATION_FUNCTIONS: readonly ["t", "msg", "declare_static", "declare_var"];
+export declare const PYTHON_METADATA_KWARGS: {
+    readonly _id: "id";
+    readonly _context: "context";
+    readonly _max_chars: "maxChars";
+};

package/dist/constants.js

@@ -0,0 +1,18 @@
+export const PYTHON_GT_PACKAGES = ['gt_flask', 'gt_fastapi'];
+export const PYTHON_GT_DEPENDENCIES = ['gt-flask', 'gt-fastapi'];
+export const PYTHON_T_FUNCTION = 't';
+export const PYTHON_MSG_FUNCTION = 'msg';
+export const PYTHON_DECLARE_STATIC = 'declare_static';
+export const PYTHON_DECLARE_VAR = 'declare_var';
+/** These imported names are tracked (translation functions + static helpers) */
+export const PYTHON_TRANSLATION_FUNCTIONS = [
+    't',
+    'msg',
+    'declare_static',
+    'declare_var',
+];
+export const PYTHON_METADATA_KWARGS = {
+    _id: 'id',
+    _context: 'context',
+    _max_chars: 'maxChars',
+};

package/dist/extractCalls.d.ts

@@ -0,0 +1,20 @@
+import type { SyntaxNode } from './parser.js';
+import type { ImportAlias } from './extractImports.js';
+export type RawTranslationCall = {
+    source: string;
+    id?: string;
+    context?: string;
+    maxChars?: number;
+    staticId?: string;
+    line: number;
+    column: number;
+};
+/**
+ * Extracts translation function calls from a Python AST.
+ * Walks all `call` nodes and checks if they reference a tracked import.
+ */
+export declare function extractCalls(rootNode: SyntaxNode, imports: ImportAlias[], filePath: string): Promise<{
+    calls: RawTranslationCall[];
+    errors: string[];
+    warnings: string[];
+}>;

package/dist/extractCalls.js

@@ -0,0 +1,211 @@
+import { PYTHON_METADATA_KWARGS } from './constants.js';
+import { containsStaticCalls, parseStringExpression, } from './parseStringExpression.js';
+import { nodeToStrings } from './stringNode.js';
+import { indexVars } from 'generaltranslation/internal';
+import { randomUUID } from 'node:crypto';
+/**
+ * Extracts translation function calls from a Python AST.
+ * Walks all `call` nodes and checks if they reference a tracked import.
+ */
+export async function extractCalls(rootNode, imports, filePath) {
+    const calls = [];
+    const errors = [];
+    const warnings = [];
+    // Only track t/msg as translation functions (not declare_static/declare_var)
+    const trackedNames = new Set(imports
+        .filter((imp) => imp.originalName !== 'declare_static' &&
+        imp.originalName !== 'declare_var')
+        .map((imp) => imp.localName));
+    if (trackedNames.size === 0)
+        return { calls, errors, warnings };
+    await walkCalls(rootNode, trackedNames, imports, filePath, calls, errors, warnings);
+    return { calls, errors, warnings };
+}
+async function walkCalls(node, trackedNames, imports, filePath, calls, errors, warnings) {
+    if (node.type === 'call') {
+        const funcNode = node.childForFieldName('function');
+        if (funcNode &&
+            funcNode.type === 'identifier' &&
+            trackedNames.has(funcNode.text)) {
+            await processCall(node, imports, filePath, calls, errors, warnings);
+        }
+    }
+    for (let i = 0; i < node.childCount; i++) {
+        const child = node.child(i);
+        if (child)
+            await walkCalls(child, trackedNames, imports, filePath, calls, errors, warnings);
+    }
+}
+async function processCall(callNode, imports, filePath, calls, errors, _warnings) {
+    const argsNode = callNode.childForFieldName('arguments');
+    if (!argsNode) {
+        errors.push(`${locationStr(callNode)}: translation call has no arguments`);
+        return;
+    }
+    // Find first positional argument (skip punctuation)
+    let firstArg = null;
+    for (let i = 0; i < argsNode.childCount; i++) {
+        const child = argsNode.child(i);
+        if (child &&
+            child.type !== '(' &&
+            child.type !== ')' &&
+            child.type !== ',' &&
+            child.type !== 'keyword_argument') {
+            firstArg = child;
+            break;
+        }
+    }
+    if (!firstArg) {
+        errors.push(`${locationStr(callNode)}: translation call has no positional argument`);
+        return;
+    }
+    // Check if this expression contains declare_static/declare_var
+    const hasStaticHelpers = (firstArg.type === 'string' &&
+        isFString(firstArg) &&
+        containsStaticCalls(firstArg, imports)) ||
+        (firstArg.type === 'binary_operator' &&
+            containsStaticCalls(firstArg, imports)) ||
+        (firstArg.type === 'call' && containsStaticCalls(firstArg, imports));
+    if (hasStaticHelpers) {
+        // Compound expression path: parse into StringNode tree
+        const rootNode = callNode.tree?.rootNode;
+        if (!rootNode) {
+            errors.push(`${locationStr(callNode)}: could not access AST root`);
+            return;
+        }
+        const stringNode = await parseStringExpression(firstArg, {
+            rootNode,
+            imports,
+            filePath,
+            errors,
+        });
+        if (!stringNode)
+            return;
+        const strings = nodeToStrings(stringNode).map(indexVars);
+        if (strings.length === 0) {
+            errors.push(`${locationStr(callNode)}: no string variants produced`);
+            return;
+        }
+        const metadata = extractKwargs(argsNode, errors, callNode);
+        const staticId = `static-temp-id-${randomUUID()}`;
+        for (const source of strings) {
+            calls.push({
+                source,
+                ...metadata,
+                staticId,
+                line: callNode.startPosition.row + 1,
+                column: callNode.startPosition.column,
+            });
+        }
+        return;
+    }
+    // Simple path: validate first argument is a plain string literal
+    if (firstArg.type !== 'string') {
+        if (firstArg.type === 'identifier') {
+            errors.push(`${locationStr(callNode)}: translation call uses a variable "${firstArg.text}" instead of a string literal`);
+        }
+        else if (firstArg.type === 'concatenated_string') {
+            errors.push(`${locationStr(callNode)}: translation call uses concatenated strings — use a single string literal`);
+        }
+        else {
+            errors.push(`${locationStr(callNode)}: translation call first argument must be a string literal, got "${firstArg.type}"`);
+        }
+        return;
+    }
+    // Check for f-strings (without declare_static/declare_var)
+    if (isFString(firstArg)) {
+        errors.push(`${locationStr(callNode)}: translation call uses an f-string — use a plain string literal or declare_static()/declare_var()`);
+        return;
+    }
+    const source = extractStringContent(firstArg);
+    if (source === undefined) {
+        errors.push(`${locationStr(callNode)}: could not extract string content`);
+        return;
+    }
+    // Extract keyword arguments
+    const metadata = extractKwargs(argsNode, errors, callNode);
+    calls.push({
+        source,
+        ...metadata,
+        line: callNode.startPosition.row + 1,
+        column: callNode.startPosition.column,
+    });
+}
+function extractKwargs(argsNode, errors, callNode) {
+    const result = {};
+    for (let i = 0; i < argsNode.childCount; i++) {
+        const child = argsNode.child(i);
+        if (!child || child.type !== 'keyword_argument')
+            continue;
+        const nameNode = child.childForFieldName('name');
+        const valueNode = child.childForFieldName('value');
+        if (!nameNode || !valueNode)
+            continue;
+        const kwargName = nameNode.text;
+        const metadataKey = PYTHON_METADATA_KWARGS[kwargName];
+        if (!metadataKey)
+            continue;
+        if (metadataKey === 'maxChars') {
+            if (valueNode.type === 'integer') {
+                result.maxChars = parseInt(valueNode.text, 10);
+            }
+            else {
+                errors.push(`${locationStr(callNode)}: _max_chars must be an integer literal`);
+            }
+        }
+        else {
+            if (valueNode.type === 'string' && !isFString(valueNode)) {
+                const value = extractStringContent(valueNode);
+                if (value !== undefined) {
+                    if (metadataKey === 'id')
+                        result.id = value;
+                    else if (metadataKey === 'context')
+                        result.context = value;
+                }
+            }
+            else {
+                errors.push(`${locationStr(callNode)}: _${metadataKey} must be a string literal`);
+            }
+        }
+    }
+    return result;
+}
+function isFString(stringNode) {
+    // Check if string_start begins with 'f' or 'F'
+    for (let i = 0; i < stringNode.childCount; i++) {
+        const child = stringNode.child(i);
+        if (child && child.type === 'string_start') {
+            return /^[fF]/.test(child.text);
+        }
+        // Also check for interpolation children (hallmark of f-strings)
+        if (child && child.type === 'interpolation') {
+            return true;
+        }
+    }
+    return false;
+}
+function extractStringContent(stringNode) {
+    // Look for string_content child
+    for (let i = 0; i < stringNode.childCount; i++) {
+        const child = stringNode.child(i);
+        if (child && child.type === 'string_content') {
+            return child.text;
+        }
+    }
+    // Empty string — no string_content child, but has string_start and string_end
+    let hasStart = false;
+    let hasEnd = false;
+    for (let i = 0; i < stringNode.childCount; i++) {
+        const child = stringNode.child(i);
+        if (child?.type === 'string_start')
+            hasStart = true;
+        if (child?.type === 'string_end')
+            hasEnd = true;
+    }
+    if (hasStart && hasEnd)
+        return '';
+    return undefined;
+}
+function locationStr(node) {
+    return `line ${node.startPosition.row + 1}, col ${node.startPosition.column}`;
+}

package/dist/extractImports.d.ts

@@ -0,0 +1,18 @@
+import type { SyntaxNode } from './parser.js';
+export type ImportAlias = {
+    /** The local name used in the source file (e.g. "translate" for `import t as translate`) */
+    localName: string;
+    /** The original imported name (e.g. "t") */
+    originalName: string;
+    /** The package it was imported from (e.g. "gt_flask") */
+    packageName: string;
+};
+/**
+ * Extracts GT-related imports from a Python AST.
+ *
+ * Handles:
+ * - `from gt_flask import t`
+ * - `from gt_flask import t as translate`
+ * - `from gt_flask import t, msg`
+ */
+export declare function extractImports(rootNode: SyntaxNode): ImportAlias[];

package/dist/extractImports.js

@@ -0,0 +1,86 @@
+import { PYTHON_GT_PACKAGES, PYTHON_TRANSLATION_FUNCTIONS, } from './constants.js';
+/**
+ * Extracts GT-related imports from a Python AST.
+ *
+ * Handles:
+ * - `from gt_flask import t`
+ * - `from gt_flask import t as translate`
+ * - `from gt_flask import t, msg`
+ */
+export function extractImports(rootNode) {
+    const aliases = [];
+    for (let i = 0; i < rootNode.childCount; i++) {
+        const node = rootNode.child(i);
+        if (!node || node.type !== 'import_from_statement')
+            continue;
+        const moduleName = getModuleName(node);
+        if (!moduleName || !isGtPackage(moduleName))
+            continue;
+        // Collect all imported names from this statement
+        for (let j = 0; j < node.childCount; j++) {
+            const child = node.child(j);
+            if (!child)
+                continue;
+            if (child.type === 'aliased_import') {
+                // `from gt_flask import t as translate`
+                const nameNode = child.childForFieldName('name');
+                const aliasNode = child.childForFieldName('alias');
+                const originalName = nameNode ? getIdentifierText(nameNode) : undefined;
+                const localName = aliasNode ? aliasNode.text : originalName;
+                if (originalName && localName && isTranslationFunction(originalName)) {
+                    aliases.push({ localName, originalName, packageName: moduleName });
+                }
+            }
+            else if (child.type === 'dotted_name') {
+                // Skip the module name itself (first dotted_name is the module)
+                const text = child.text;
+                if (text === moduleName)
+                    continue;
+                // `from gt_flask import t` — only track translation functions
+                if (isTranslationFunction(text)) {
+                    aliases.push({
+                        localName: text,
+                        originalName: text,
+                        packageName: moduleName,
+                    });
+                }
+            }
+        }
+    }
+    return aliases;
+}
+function getModuleName(importNode) {
+    const moduleNode = importNode.childForFieldName('module_name');
+    if (moduleNode)
+        return moduleNode.text;
+    // Fallback: find the first dotted_name child (before 'import' keyword)
+    for (let i = 0; i < importNode.childCount; i++) {
+        const child = importNode.child(i);
+        if (!child)
+            continue;
+        if (child.type === 'import')
+            break; // reached the 'import' keyword
+        if (child.type === 'dotted_name')
+            return child.text;
+    }
+    return undefined;
+}
+function getIdentifierText(node) {
+    if (node.type === 'identifier')
+        return node.text;
+    if (node.type === 'dotted_name') {
+        // Get the last identifier in a dotted name
+        for (let i = node.childCount - 1; i >= 0; i--) {
+            const child = node.child(i);
+            if (child && child.type === 'identifier')
+                return child.text;
+        }
+    }
+    return node.text;
+}
+function isGtPackage(name) {
+    return PYTHON_GT_PACKAGES.includes(name);
+}
+function isTranslationFunction(name) {
+    return PYTHON_TRANSLATION_FUNCTIONS.includes(name);
+}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+import type { ExtractionResult } from './types.js';
+export type { ExtractionResult, ExtractionMetadata } from './types.js';
+export type { ImportAlias } from './extractImports.js';
+export { PYTHON_GT_PACKAGES, PYTHON_GT_DEPENDENCIES, PYTHON_T_FUNCTION, PYTHON_MSG_FUNCTION, PYTHON_DECLARE_STATIC, PYTHON_DECLARE_VAR, PYTHON_TRANSLATION_FUNCTIONS, PYTHON_METADATA_KWARGS, } from './constants.js';
+export declare function extractFromPythonSource(sourceCode: string, filePath: string): Promise<{
+    results: ExtractionResult[];
+    errors: string[];
+    warnings: string[];
+}>;

package/dist/index.js

@@ -0,0 +1,42 @@
+import { getParser } from './parser.js';
+import { extractImports } from './extractImports.js';
+import { extractCalls } from './extractCalls.js';
+export { PYTHON_GT_PACKAGES, PYTHON_GT_DEPENDENCIES, PYTHON_T_FUNCTION, PYTHON_MSG_FUNCTION, PYTHON_DECLARE_STATIC, PYTHON_DECLARE_VAR, PYTHON_TRANSLATION_FUNCTIONS, PYTHON_METADATA_KWARGS, } from './constants.js';
+export async function extractFromPythonSource(sourceCode, filePath) {
+    const parser = await getParser();
+    const tree = parser.parse(sourceCode);
+    if (!tree) {
+        return {
+            results: [],
+            errors: [`Failed to parse ${filePath}`],
+            warnings: [],
+        };
+    }
+    // Step 1: Extract GT imports
+    const imports = extractImports(tree.rootNode);
+    if (imports.length === 0) {
+        return { results: [], errors: [], warnings: [] };
+    }
+    // Step 2: Extract translation calls
+    const { calls, errors, warnings } = await extractCalls(tree.rootNode, imports, filePath);
+    // Step 3: Map to ExtractionResult
+    const results = calls.map((call) => ({
+        dataFormat: 'ICU',
+        source: call.source,
+        metadata: {
+            ...(call.id && { id: call.id }),
+            ...(call.context && { context: call.context }),
+            ...(call.maxChars != null && { maxChars: call.maxChars }),
+            ...(call.staticId && { staticId: call.staticId }),
+            filePaths: [filePath],
+        },
+    }));
+    return {
+        results,
+        errors: prefixErrors(errors, filePath),
+        warnings: prefixErrors(warnings, filePath),
+    };
+}
+function prefixErrors(messages, filePath) {
+    return messages.map((msg) => `${filePath}: ${msg}`);
+}

package/dist/parseStringExpression.d.ts

@@ -0,0 +1,20 @@
+import type { SyntaxNode } from './parser.js';
+import type { StringNode } from './stringNode.js';
+import type { ImportAlias } from './extractImports.js';
+type ParseContext = {
+    rootNode: SyntaxNode;
+    imports: ImportAlias[];
+    filePath: string;
+    errors: string[];
+};
+/**
+ * Checks if an expression contains declare_static or declare_var calls.
+ */
+export declare function containsStaticCalls(node: SyntaxNode, imports: ImportAlias[]): boolean;
+/**
+ * Parses the first argument of t() into a StringNode tree.
+ * Handles: plain strings, f-strings with declare_static/declare_var,
+ * binary + concatenation, and standalone declare_static calls.
+ */
+export declare function parseStringExpression(node: SyntaxNode, ctx: ParseContext): Promise<StringNode | null>;
+export {};