@apify/docusaurus-plugin-typedoc-api 4.2.5 → 4.2.6

package/src/plugin/python-generator/index.ts

@@ -1,18 +0,0 @@
-import { spinner } from 'zx';
-import { parseWithPydocMarkdown } from "./pydoc-markdown";
-import { pydocToTypedoc } from "./transform-docs";
-
-export async function generateJsonFromPythonProject({
-    outFile,
-    projectRoot,
-} : { outFile: string, projectRoot: string }): Promise<void> {  
-    const pydocJson = await parseWithPydocMarkdown({ projectRoot });
-
-    await spinner('Converting the Python JSON AST to a TypeDoc-compliant file...', async () => {
-        await pydocToTypedoc({
-            moduleName: 'python', // TODO: get from project config files or passed options
-            outFile,
-            pydocJson,
-        });
-    });
-}

package/src/plugin/python-generator/pydoc-markdown.ts

@@ -1,70 +0,0 @@
-import { rmSync, writeFileSync } from 'fs';
-import path from 'path';
-import { $, ProcessOutput, spinner } from 'zx';
-
-/**
- * Generates the pydoc-markdown configuration file
- * @returns The pydoc-markdown configuration file as a string
- */
-function getConfigYml({
-    projectRoot
-}: { projectRoot: string }): string {
-    return `
-loaders:
-  - type: python
-    search_path: ["${projectRoot}"]
-processors:
-  - type: filter
-    skip_empty_modules: true
-  - type: crossref
-renderer:
-  type: docusaurus
-  docs_base_path: docs
-  relative_output_path: reference
-  relative_sidebar_path: sidebar.json
-  sidebar_top_level_label: null
-`
-}
-
-export async function parseWithPydocMarkdown({
-    projectRoot,
-}: {
-    projectRoot: string,
-}
-): Promise<string> {
-    // Check whether the user has Python and pydoc-markdown installed
-    for (const cmd of ['python', 'pydoc-markdown']) {
-        try {
-            // eslint-disable-next-line no-await-in-loop
-            await spinner(`Checking for ${cmd}...`, () => $`${cmd} --version`);
-        } catch {
-            throw new Error(`Please install ${cmd} to use this plugin with Python projects.`);
-        }
-    };
-
-    // Generate the JSON file
-    try {
-        const configYml = getConfigYml({ projectRoot });
-        const configPath = path.join(__dirname, 'pydoc-markdown.temp.yml');
-        writeFileSync(configPath, configYml);
-
-        let pydoc: ProcessOutput | null = null;
-
-        await spinner('Parsing the Python project into a JSON AST...', async () => {
-            pydoc = await $`pydoc-markdown --quiet --dump ${configPath}`;
-        });
-
-        rmSync(configPath);
-
-        // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
-        let json = await pydoc!.text();
-    
-        json = json.replaceAll(path.resolve(projectRoot), 'REPO_ROOT_PLACEHOLDER');
-    
-        return json;
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    } catch (error: any) {
-        // eslint-disable-next-line 
-        throw new Error(`Failed to generate JSON file from Python project:\n\t${error.stderr.split('\n').slice(-2).join('\n')}`);
-    }
-}

package/src/plugin/python-generator/transform-docs.ts

@@ -1,417 +0,0 @@
-/* eslint-disable */ 
-import fs from 'fs';
-import { $ } from 'zx';
-
-const REPO_ROOT_PLACEHOLDER = 'REPO_ROOT_PLACEHOLDER';
-
-// TODO: Make these parametrizable (gitRepoUrls array option)
-const APIFY_CLIENT_REPO_URL = 'https://github.com/apify/apify-client-python';
-const APIFY_SDK_REPO_URL    = 'https://github.com/apify/apify-sdk-python';
-const APIFY_SHARED_REPO_URL = 'https://github.com/apify/apify-shared-python';
-const CRAWLEE_PYTHON_REPO_URL = 'https://github.com/apify/crawlee-python';
-
-const REPO_URL_PER_PACKAGE = {
-    'apify': APIFY_SDK_REPO_URL,
-    'apify_client': APIFY_CLIENT_REPO_URL,
-    'apify_shared': APIFY_SHARED_REPO_URL,
-    'crawlee': CRAWLEE_PYTHON_REPO_URL,
-} as const;
-
-const TAG_PER_PACKAGE: Record<string, string> = {};
-let MODULE_SHORTCUTS: Record<string, string> = {};
-
-async function initPackageTags({
-    moduleName
-}: {
-    moduleName?: string,
-}) {
-    // For each package, get the installed version, and set the tag to the corresponding version
-    for (const pkg of ['apify', 'apify_client', 'apify_shared']) {
-        try {
-            const packageVersion = await $`python -c 'import ${pkg}; print(${pkg}.__version__)'`;
-            if (packageVersion.exitCode === 0) {
-                TAG_PER_PACKAGE[pkg] = `v${await packageVersion.text()}`;
-            }
-        } catch (e) {
-            console.warn(`Failed to get version of package ${pkg}`);
-        }
-    }
-
-    if(!moduleName) {
-        const thisPackagePyprojectToml = fs.readFileSync('../pyproject.toml', 'utf8');
-        moduleName = thisPackagePyprojectToml.match(/^name = "(.+)"$/m)?.[1];
-    }
-
-    // For the current package, set the tag to 'master'
-    TAG_PER_PACKAGE[moduleName!] = 'master';
-
-    return TAG_PER_PACKAGE;
-}
-
-async function initModuleShortcuts() {
-    if(!fs.existsSync('./module_shortcuts.json')) {
-        return console.warn('No module_shortcuts.json file found, skipping module shortcuts.');
-    }
-    
-    MODULE_SHORTCUTS = JSON.parse(fs.readFileSync('./module_shortcuts.json', 'utf8'));
-}
-
-// Taken from https://github.com/TypeStrong/typedoc/blob/v0.23.24/src/lib/models/reflections/kind.ts, modified
-const TYPEDOC_KINDS = {
-    'class': {
-        kind: 128,
-        kindString: 'Class',
-    },
-    'function': {
-        kind: 2048,
-        kindString: 'Method',
-    },
-    'data': {
-        kind: 1024,
-        kindString: 'Property',
-    },
-    'enum': {
-        kind: 8,
-        kindString: 'Enumeration',
-    },
-    'enumValue': {
-        kind: 16,
-        kindString: 'Enumeration Member',
-    },
-}
-
-const GROUP_ORDER = [
-    'Main Classes',
-    'Helper Classes',
-    'Errors',
-    'Constructors',
-    'Methods',
-    'Properties',
-    'Constants',
-    'Enumeration Members'
-] as const;
-
-const groupSort = (g1: typeof GROUP_ORDER[number], g2: typeof GROUP_ORDER[number]) => {
-    if(GROUP_ORDER.includes(g1) && GROUP_ORDER.includes(g2)){
-        return GROUP_ORDER.indexOf(g1) - GROUP_ORDER.indexOf(g2)
-    }
-    return g1.localeCompare(g2);
-};
-
-function getGroupName(object: any) {
-    const groupPredicates: Record<
-        typeof GROUP_ORDER[number],  
-        (object: any) => boolean
-    > = {
-        'Errors': (x) => x.name.toLowerCase().includes('error'),
-        'Main Classes': (x) => ['Dataset', 'KeyValueStore', 'RequestQueue'].includes(x.name) || x.name.endsWith('Crawler'),
-        'Helper Classes': (x) => x.kindString === 'Class',
-        'Methods': (x) => x.kindString === 'Method',
-        'Constructors': (x) => x.kindString === 'Constructor',
-        'Properties': (x) => x.kindString === 'Property',
-        'Constants': (x) => x.kindString === 'Enumeration',
-        'Enumeration Members': (x) => x.kindString === 'Enumeration Member',
-    };
-
-    const [group] = Object.entries(groupPredicates).find(
-        ([_, predicate]) => predicate(object)
-    )!;
-
-    return group;
-}
-
-// Strips the Optional[] type from the type string, and replaces generic types with just the main type
-function getBaseType(type: any) {
-    return type?.replace(/Optional\[(.*)\]/g, '$1').replace('ListPage[Dict]', 'ListPage');
-}
-
-// Returns whether a type is a custom class, or a primitive type
-function isCustomClass(type: string) {
-    return !['dict', 'list', 'str', 'int', 'float', 'bool'].includes(type.toLowerCase());
-}
-
-// Infer the Typedoc type from the docspec type
-function inferTypedocType(docspecType: any): Record<string, any> | undefined {
-    const typeWithoutOptional = getBaseType(docspecType);
-    if (!typeWithoutOptional) {
-        return undefined;
-    }
-
-    // Typically, if a type is a custom class, it will be a reference in Typedoc
-    return isCustomClass(typeWithoutOptional) ? {
-        type: 'reference',
-        name: docspecType
-    } : {
-        type: 'intrinsic',
-        name: docspecType,
-    }
-}
-
-// Sorts the groups of a Typedoc member, and sorts the children of each group
-function sortChildren(typedocMember: any) {
-    for (let group of typedocMember.groups) {
-        group.children
-            .sort((a: any, b: any) => {
-                const firstName = typedocMember.children.find((x: any) => x.id === a).name;
-                const secondName = typedocMember.children.find((x: any) => x.id === b).name;
-                return firstName.localeCompare(secondName);
-            });
-    }
-    typedocMember.groups.sort((a: { title: typeof GROUP_ORDER[number] }, b: { title: typeof GROUP_ORDER[number] }) => groupSort(a.title, b.title));
-}
-
-// Parses the arguments and return value description of a method from its docstring
-function extractArgsAndReturns(docstring: string) {
-    const parameters = (docstring
-        .split('Args:')[1] ?? '').split('Returns:')[0] // Get the part between Args: and Returns:
-        .split(/(^|\n)\s*([\w]+)\s*\(.*?\)\s*:\s*/) // Magic regex which splits the arguments into an array, and removes the argument types
-        .filter(x => x.length > 1) // Remove empty strings
-        .reduce((acc, curr, idx, arr) => { // Collect the argument names and types into an object
-            if(idx % 2 === 0){
-                return {...acc, [curr]: arr[idx+1]} // If the index is even, the current string is an argument name, and the next string is its type
-            }
-            return acc;
-        }, {} as Record<string, string>);
-
-    const returns = (docstring
-        .split('Returns:')[1] ?? '').split('Raises:')[0] // Get the part between Returns: and Raises:
-        .split(':')[1]?.trim() || undefined; // Split the return value into its type and description, return description
-
-
-    return { parameters, returns };
-}
-
-// Objects with decorators named 'ignore_docs' or with empty docstrings will be ignored
-function isHidden(member: any) {
-    return member.decorations?.some((d: { name: string }) => d.name === 'ignore_docs') || member.name === 'ignore_docs';
-}
-
-// Each object in the Typedoc structure has an unique ID,
-// we'll just increment it for each object we convert
-let oid = 1;
-
-const symbolIdMap: { qualifiedName: string, sourceFileName: string }[]  = [];
-
-// Converts a docspec object to a Typedoc object, including all its children
-function convertObject(obj: any, parent: any, module: any) {
-    const rootModuleName: string = module.name.split('.')[0];
-    for (let member of obj.members ?? []) {
-        let typedocKind = TYPEDOC_KINDS[member.type as keyof typeof TYPEDOC_KINDS];
-
-        if(member.bases?.includes('Enum')) {
-            typedocKind = TYPEDOC_KINDS['enum'];
-        }
-
-        let typedocType = inferTypedocType(member.datatype);
-        
-        if (member.decorations?.some((d: { name: string }) => ['property', 'dualproperty'].includes(d.name))) {
-            typedocKind = TYPEDOC_KINDS['data'];
-            typedocType = inferTypedocType(member.return_type ?? member.datatype);
-        }
-        
-        if(parent.kindString === 'Enumeration') {
-            typedocKind = TYPEDOC_KINDS['enumValue'];
-            typedocType = {
-                type: 'literal',
-                value: member.value,
-            }
-        }
-
-        if(member.type in TYPEDOC_KINDS && !isHidden(member)) {
-            // Get the URL of the member in GitHub
-            const repoBaseUrl = `${REPO_URL_PER_PACKAGE[rootModuleName as keyof typeof REPO_URL_PER_PACKAGE]}/blob/${TAG_PER_PACKAGE[rootModuleName] ?? 'master'}`;
-            const filePathInRepo = member.location.filename.replace(REPO_ROOT_PLACEHOLDER, '');
-            const fileGitHubUrl = member.location.filename.replace(REPO_ROOT_PLACEHOLDER, repoBaseUrl);
-            const memberGitHubUrl = `${fileGitHubUrl}#L${member.location.lineno}`;
-
-            symbolIdMap.push({
-                qualifiedName: member.name,
-                sourceFileName: filePathInRepo,
-            });
-
-            // Get the module name of the member, and check if it has a shortcut (reexport from an ancestor module)
-            const fullName = `${module.name}.${member.name}`;
-            let moduleName = module.name;
-            if (fullName in MODULE_SHORTCUTS) {
-                moduleName = MODULE_SHORTCUTS[fullName].replace(`.${member.name}`, '');
-            }
-
-            // Create the Typedoc member object
-            let typedocMember = {
-                id: oid++,
-                name: member.name,
-                module: moduleName, // This is an extension to the original Typedoc structure, to support showing where the member is exported from
-                ...typedocKind,
-                flags: {},
-                comment: member.docstring ? {
-                    summary: [{
-                        kind: 'text',
-                        text: member.docstring?.content,
-                    }],
-                } : undefined,
-                type: typedocType,
-                children: [],
-                groups: [],
-                sources: [{
-                    filename: filePathInRepo,
-                    line: member.location.lineno,
-                    character: 1,
-                    url: memberGitHubUrl,
-                }],
-                signatures: [] as any[],
-            };
-
-            if(typedocMember.kindString === 'Method') {
-                const { parameters, returns } = extractArgsAndReturns(member.docstring?.content ?? '');
-
-                typedocMember.signatures = [{
-                    id: oid++,
-                    name: member.name,
-                    modifiers: member.modifiers ?? [],
-                    kind: 4096,
-                    kindString: 'Call signature',
-                    flags: {},
-                    comment: member.docstring ? {
-                        summary: [{
-                            kind: 'text',
-                            text: member.docstring?.content
-                                .replace(/\**(Args|Arguments|Returns)[\s\S]+/, ''),
-                        }],
-                        blockTags: returns ? [
-                            { tag: '@returns', content: [{ kind: 'text', text: returns }] },
-                        ] : undefined,
-                    } : undefined,
-                    type: inferTypedocType(member.return_type),
-                    parameters: member.args
-                        .filter((arg: any) => (arg.name !== 'self' && arg.name !== 'cls'))
-                        .map((arg: any) => ({
-                            id: oid++,
-                            name: arg.name,
-                            kind: 32768,
-                            kindString: 'Parameter',
-                            flags: {
-                                isOptional: arg.datatype?.includes('Optional') ? 'true' : undefined,
-                                'keyword-only': arg.type === 'KEYWORD_ONLY' ? 'true' : undefined,
-                            },
-                            type: inferTypedocType(arg.datatype),
-                            comment: parameters[arg.name] ? {
-                                summary: [{
-                                    kind: 'text',
-                                    text: parameters[arg.name]
-                                }]
-                            } : undefined,
-                            defaultValue: arg.default_value,
-                        })),
-                }];
-            }
-
-            if(typedocMember.name === '__init__') {
-                typedocMember.kind = 512;
-                typedocMember.kindString = 'Constructor';
-            }
-
-            convertObject(member, typedocMember, module);
-
-            const groupName = getGroupName(typedocMember);
-
-            const group = parent.groups.find((g: { title: string }) => g.title === groupName);
-            if (group) {
-                group.children.push(typedocMember.id);
-            } else {
-                parent.groups.push({
-                    title: groupName,
-                    children: [typedocMember.id],
-                });
-            }
-
-            sortChildren(typedocMember);
-            parent.children.push(typedocMember);
-        }
-    }
-}
-
-export async function pydocToTypedoc({
-    pydocFile,
-    pydocJson,
-    outFile,
-    moduleName,
-}: { 
-    pydocFile?: string,
-    pydocJson?: string,
-    outFile: string,
-    moduleName?: string,
-}) {
-    await initPackageTags({ moduleName });
-    await initModuleShortcuts();
-
-    // Root object of the Typedoc structure
-    const typedocApiReference = {
-        'id': 0,
-        'name': 'apify-client',
-        'kind': 1,
-        'kindString': 'Project',
-        'flags': {},
-        'originalName': '',
-        'children': [],
-        'groups': [],
-        'sources': [
-            {
-                'fileName': 'src/index.ts',
-                'line': 1,
-                'character': 0,
-                'url': `http://example.com/blob/123456/src/dummy.py`,
-            }
-        ],
-        'symbolIdMap': {},
-    };
-
-    // Load the docspec dump files of this module and of apify-shared
-    const thisPackageDocspecDump = pydocJson ?? fs.readFileSync(pydocFile!, 'utf8');
-    const thisPackageModules = thisPackageDocspecDump.split('\n').filter((line) => line !== '');
-
-    // Convert all the modules, store them in the root object
-    for (const module of [...thisPackageModules]) {
-        const parsedModule = JSON.parse(module);
-        convertObject(parsedModule, typedocApiReference, parsedModule);
-    };
-
-    // Recursively fix references (collect names->ids of all the named entities and then inject those in the reference objects)
-    const namesToIds: Record<string, string> = {};
-    function collectIds(obj: Record<string, any>) {
-        for (const child of obj.children ?? []) {
-            namesToIds[child.name] = child.id;
-            collectIds(child);
-        }
-    }
-    collectIds(typedocApiReference);
-
-    function fixRefs(obj: Record<string, any>) {
-        for (const child of obj.children ?? []) {
-            if (child.type?.type === 'reference') {
-                child.type.id = namesToIds[child.type.name];
-            }
-            if (child.signatures) {
-                for (const sig of child.signatures) {
-                    for (const param of sig.parameters ?? []) {
-                        if (param.type?.type === 'reference') {
-                            param.type.id = namesToIds[param.type.name];
-                        }
-                    }
-                    if (sig.type?.type === 'reference') {
-                        sig.type.id = namesToIds[sig.type.name];
-                    }
-                }
-            }
-            fixRefs(child);
-        }
-    }
-    fixRefs(typedocApiReference);
-
-    // Sort the children of the root object
-    sortChildren(typedocApiReference);
-
-    typedocApiReference.symbolIdMap = Object.fromEntries(Object.entries(symbolIdMap));
-
-    // Write the Typedoc structure to the output file
-    fs.writeFileSync(outFile, JSON.stringify(typedocApiReference, null, 4));
-}