@apify/docusaurus-plugin-typedoc-api 4.2.10 → 4.2.11-1

package/src/plugin/python/type-parsing/parse_types.py

@@ -0,0 +1,82 @@
+"""
+Given a JSON file containing a list of expressions, this script will parse each expression and output a JSON file containing an object with the parsed expressions.
+Called from transformDocs.js.
+
+Accepts one CLI argument: the path to the JSON file containing the expressions to parse.
+"""
+
+import ast
+import json
+import sys
+import os
+
+base_scalar_types = {
+    "str",
+    "int",
+    "float",
+    "bool",
+    "bytearray",
+    "timedelta",
+    "None",
+}
+
+
+def parse_expression(ast_node, full_expression):
+    """
+    Turns the AST expression object into a typedoc-compliant dict
+    """
+
+    current_node_type = ast_node.__class__.__name__
+
+    if current_node_type == "BinOp" and ast_node.op.__class__.__name__ == "BitOr":
+        return {
+            "type": "union",
+            "types": [
+                parse_expression(ast_node.left, full_expression),
+                parse_expression(ast_node.right, full_expression),
+            ],
+        }
+
+    if current_node_type == "Tuple":
+        return [parse_expression(e, full_expression) for e in ast_node.elts]
+
+    if current_node_type == "Subscript":
+        if "id" in ast_node.value._fields and ast_node.value.id == "Annotated":
+            return parse_expression(ast_node.slice.dims[0], full_expression)
+
+        main_type = parse_expression(ast_node.value, full_expression)
+        type_argument = parse_expression(ast_node.slice, full_expression)
+
+        main_type["typeArguments"] = (
+            type_argument if isinstance(type_argument, list) else [type_argument]
+        )
+        return main_type
+
+    if current_node_type == "Constant":
+        return {"type": "literal", "value": ast_node.value}
+
+    # If the expression is not one of the types above, we simply print the expression
+    return {
+        "type": "reference",
+        "name": full_expression[ast_node.col_offset : ast_node.end_col_offset],
+    }
+
+
+typedoc_types_path = sys.argv[1]
+
+with open(typedoc_types_path, "r") as f:
+    typedoc_out = {}
+    expressions = json.load(f)
+
+    for expression in expressions:
+        try:
+            if typedoc_out.get(expression) is None:
+                typedoc_out[expression] = parse_expression(
+                    ast.parse(expression).body[0].value, expression
+                )
+        except Exception as e:
+            print(f"Invalid expression encountered while parsing: {expression}")
+            print(f"Error: {e}")
+
+    with open(f"{os.path.splitext(typedoc_types_path)[0]}-parsed.json", "w") as f:
+        f.write(json.dumps(typedoc_out, indent=4))

package/src/plugin/python/types.ts

@@ -0,0 +1,83 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { TYPEDOC_KINDS } from './consts';
+
+export type OID = number;
+
+export interface TypeDocObject {
+	[key: string]: any;
+	id: OID;
+	name: string;
+	kind: number;
+	kindString: string;
+	decorations?: { name: string; args: string }[];
+	children?: TypeDocObject[];
+	groups?: { title: string; children: OID[] }[];
+	flags: Record<string, boolean>;
+	module?: string;
+	inheritedFrom?: {
+		type: string;
+		target: OID;
+		name: string;
+	};
+	comment?: {
+		summary: { text: string; kind: 'text' }[];
+		blockTags?: { tag: string; content: any[] }[];
+	};
+	signatures?: TypeDocObject[];
+	sources?: {
+		fileName: string;
+		line: number;
+		character: number;
+	}[];
+	type?: TypeDocType;
+	symbolIdMap?: Record<number, { qualifiedName: string; sourceFileName: string }>;
+	extendedTypes?: TypeDocType[];
+	extendedBy?: TypeDocType[];
+	modifiers?: any[];
+	parameters?: TypeDocObject[];
+}
+
+export interface DocspecObject {
+	members: DocspecObject[];
+	name: string;
+	type: keyof typeof TYPEDOC_KINDS;
+	location: {
+		filename: string;
+		lineno: number;
+	};
+	decorations?: { name: string; args: string }[];
+	bases?: DocspecType[];
+	datatype?: DocspecType;
+	return_type?: DocspecType;
+	value?: any;
+	docstring?: { content: string };
+	modifiers?: DocspecType[];
+	args?: { name: string; type: DocspecType; default_value: any; datatype: DocspecType }[];
+}
+
+export interface TypeDocDocstring {
+	text: string;
+	returns?: string;
+	args?: Record<string, string>;
+	sections?: Record<string, any[]>[];
+}
+
+export interface DocspecDocstring {
+	text: string;
+	returns?: string;
+	args?: { param: string; desc: string }[];
+	sections?: Record<string, any[]>[];
+}
+
+export type TypeDocType =
+	{
+			[key: string]: any;
+			type: 'reference';
+			name: string;
+			target?: number;
+	  } | {
+			type: 'literal';
+			value: any;
+	  };
+
+export type DocspecType = string;

package/src/plugin/python/utils.ts

@@ -0,0 +1,123 @@
+/* eslint-disable sort-keys */
+import { GROUP_ORDER, TYPEDOC_KINDS } from './consts';
+import type { DocspecObject, OID, TypeDocObject } from './types';
+
+function* generateOID() {
+	let id = 1;
+	while (true) {
+		yield id++;
+	}
+}
+
+const oidGenerator = generateOID();
+
+/**
+ * Returns automatically incrementing OID. Every call to this function will return a new unique OID.
+ * @returns {number} The OID.
+ */
+export function getOID(): OID {
+	return oidGenerator.next().value as OID;
+}
+
+/**
+ * Given a TypeDoc object, returns the name of the group this object belongs to.
+ * @param object The TypeDoc object.
+ * @returns The group name and the source of the group name (either 'decorator' or 'predicate').
+ */
+export function getGroupName(object: TypeDocObject): {
+	groupName: string | undefined;
+	source: 'decorator' | 'predicate';
+} {
+	if (object.decorations?.some((d) => d.name === 'docs_group')) {
+		const parsedGroupName = object.decorations
+			.find((d) => d.name === 'docs_group')
+			?.args.slice(2, -2);
+
+		if (parsedGroupName) {
+			return {
+				groupName: parsedGroupName,
+				source: 'decorator',
+			};
+		}
+	}
+
+	const groupPredicates: Record<string, (obj: TypeDocObject) => boolean> = {
+		'Scrapy integration': (x) =>
+			[
+				'ApifyScheduler',
+				'ActorDatasetPushPipeline',
+				'ApifyHttpProxyMiddleware',
+				'apply_apify_settings',
+			].includes(x.name),
+		'Data structures': (x) =>
+			Boolean(['BaseModel', 'TypedDict'].some((base) =>
+				(x?.bases as { includes: (x: string) => boolean })?.includes(base),
+			) || x?.decorations?.some((d) => d.name === 'dataclass')),
+		Errors: (x) => x.name.toLowerCase().includes('error'),
+		Classes: (x) => x.kindString === 'Class',
+		'Main Clients': (x) => ['ApifyClient', 'ApifyClientAsync'].includes(x.name),
+		'Async Resource Clients': (x) => x.name.toLowerCase().includes('async'),
+		'Resource Clients': (x) => x.kindString === 'Class' && x.name.toLowerCase().includes('client'),
+		Methods: (x) => x.kindString === 'Method',
+		Constructors: (x) => x.kindString === 'Constructor',
+		Properties: (x) => x.kindString === 'Property',
+		Constants: (x) => x.kindString === 'Enumeration',
+		'Enumeration members': (x) => x.kindString === 'Enumeration Member',
+	};
+
+	const groupName = Object.entries(groupPredicates).find(([_, predicate]) =>
+		predicate(object),
+	)?.[0];
+
+	return { groupName, source: 'predicate' };
+}
+
+/**
+ * Recursively search arbitrary JS object for property `name: 'docs_group'`.
+ * @param object
+ */
+export function projectUsesDocsGroupDecorator(object: { name: string }): boolean {
+	if (object instanceof Object) {
+		if (object.name === 'docs_group') {
+			return true;
+		}
+
+		for (const key in object) {
+			if (projectUsesDocsGroupDecorator(object[key as keyof typeof object] as unknown as { name: string })) {
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+/**
+ * Returns true if the given member should be hidden from the documentation.
+ *
+ * A member should be hidden if:
+ * - It has a `ignore_docs` decoration.
+ *
+ * @param member The member to check.
+ */
+export function isHidden(member: DocspecObject): boolean {
+	return (
+		!(member.type in TYPEDOC_KINDS) ||
+		member.decorations?.some((d) => d.name === 'ignore_docs') ||
+		member.name === 'ignore_docs'
+	);
+}
+
+/**
+ * Comparator for enforcing the documentation groups order (examples of groups in {@link GROUP_ORDER}).
+ *
+ * The groups are sorted by the order in which they appear in {@link GROUP_ORDER}.
+ *
+ * This is compatible with the `Array.prototype.sort` method.
+ */
+export function groupSort(g1: string, g2: string) {
+	if (GROUP_ORDER.includes(g1) && GROUP_ORDER.includes(g2)) {
+		return GROUP_ORDER.indexOf(g1) - GROUP_ORDER.indexOf(g2);
+	}
+	return g1.localeCompare(g2);
+}

package/src/plugin/structure/0.23.ts

@@ -1,5 +1,3 @@
-/* eslint-disable no-param-reassign */
-
 import { JSONOutput } from 'typedoc';
 
 interface OldComment {

package/src/plugin/version.ts

@@ -46,9 +46,9 @@ function createVersionMetadata({
 	const isLast = versionName === lastVersionName;
 	const versionOptions = options.versions[versionName] ?? {};
 	const versionLabel =
-		versionOptions.label ?? versionName === CURRENT_VERSION_NAME ? 'Next' : versionName;
+		(versionOptions.label ?? versionName === CURRENT_VERSION_NAME) ? 'Next' : versionName;
 	let versionPathPart =
-		versionOptions.path ?? versionName === CURRENT_VERSION_NAME ? 'next' : versionName;
+		(versionOptions.path ?? versionName === CURRENT_VERSION_NAME) ? 'next' : versionName;
 
 	if (isLast) {
 		versionPathPart = '';

package/src/types.ts

@@ -21,6 +21,9 @@ export interface DocusaurusPluginTypeDocApiOptions
 	minimal?: boolean;
 	packageJsonName?: string;
 	packages: (PackageConfig | string)[];
+	/**
+	 * @deprecated Use `pythonOptions` and the bundled transformation script instead.
+	 */
 	pathToCurrentVersionTypedocJSON?: string;
 	projectRoot: string;
 	readmeName?: string;
@@ -33,8 +36,13 @@ export interface DocusaurusPluginTypeDocApiOptions
 
 	/**
 	 * Enables the Python-specific rendering patches.
+	 * If `pythonOptions` is specified, this is automatically set to `true`.
 	 */
 	python: boolean;
+	pythonOptions: {
+		moduleShortcutsPath?: string;
+		pythonModulePath?: string;
+	};
 
 	remarkPlugins: MDXPlugin[];
 	rehypePlugins: MDXPlugin[];
@@ -43,6 +51,7 @@ export interface DocusaurusPluginTypeDocApiOptions
 	disableVersioning?: boolean;
 	includeCurrentVersion?: boolean;
 	routeBasePath?: string;
+	reexports: { url: string, group?: string }[];
 }
 
 export interface GlobalData {

package/src/utils/icons.ts

@@ -28,10 +28,12 @@ const KIND_ICONS: Record<ReflectionKind, string> = {
 	1_048_576: 'symbol-field', // SetSignature
 	2_097_152: 'symbol-parameter', // TypeAlias
 	4_194_304: 'references', // Reference
-	8_388_608: 'references' // a Non-TS document (new in TypeDoc `0.26.0`, unused by `docusaurus-plugin-typedoc-api`)
+	8_388_608: 'references', // a Non-TS document (new in TypeDoc `0.26.0`, unused by `docusaurus-plugin-typedoc-api`)
 };
 
-export function getKindIcon(kind: ReflectionKind, name: string): string {
+export function getKindIcon(kind: ReflectionKind, name?: string): string | null {
+	if (!name) return null;
+
 	let icon = KIND_ICONS[kind];
 
 	// Use event icon when property starts with "on"
@@ -42,7 +44,6 @@ export function getKindIcon(kind: ReflectionKind, name: string): string {
 	return icon;
 }
 
-// eslint-disable-next-line complexity
 export function getKindIconColor(kind: ReflectionKind): string {
 	switch (kind) {
 		// Function

package/src/utils/reexports.ts

@@ -0,0 +1,105 @@
+/* eslint-disable */
+// @ts-nocheck
+
+import { load } from "cheerio";
+import fs from "fs";
+
+interface TypedocJSONFile {
+    children: TypedocJSONFile[];
+    groups: { title: string, children: number[] }[];
+}
+
+interface ExternalApiItem {
+    item: TypedocJSONFile;
+    groups: string[];
+}
+
+async function loadExternalApiItem(url: string): Promise<ExternalApiItem> {
+    console.log(`Loading external API item from ${url}...`);
+    const response = await fetch(url);
+
+    const $ = load(await response.text(), { decodeEntities: false })
+    const base64encoded = $('script[type="application/typedoc-data;base64"]')?.first()?.text();
+    
+    if(!base64encoded) return null;
+    
+    const jsonData = atob(base64encoded);
+    return JSON.parse(jsonData) as ExternalApiItem;
+}
+
+// Recursively find the maximum numerical `id` property in a JS object
+function getMaxId(obj: any) {
+    let maxId = 0;
+    for (const key in obj) {
+        if (typeof obj[key] === 'object') {
+            maxId = Math.max(maxId, getMaxId(obj[key]));
+        } else if (key === 'id') {
+            maxId = Math.max(maxId, obj[key]);
+        }
+    }
+
+    return maxId;
+}
+
+function incrementIds(obj: any, increment: number): number {
+    let max = 0;
+    for (const key in obj) {
+        if (key === 'children' && Array.isArray(obj[key]) && obj[key].every((c: any) => typeof c === 'number')) {
+            obj[key] = obj[key].map((c: number) => c + increment);
+
+            max = Math.max(max, ...obj[key]);
+        } else if (key === 'id' && Number.isInteger(obj[key])) {
+            obj[key] += increment;
+            
+            max = Math.max(obj[key], max);
+        } else if (obj[key] && typeof obj[key] === 'object') {
+            max = Math.max(incrementIds(obj[key], increment), max);
+        }
+    }
+
+    return max;
+}
+
+export async function injectReexports(typedocJsonFilePath: string, reexports: { url: string, group?: string }[]): Promise<void> {
+    const typedocJson: TypedocJSONFile = await import(typedocJsonFilePath) as TypedocJSONFile;
+
+    let baseId = getMaxId(typedocJson);
+
+    const externalApiItems = await Promise.all(reexports.map(async ({url, group}) => ({
+        externalItem: await loadExternalApiItem(url),
+        customGroup: group,
+    })));
+
+    for (const { externalItem, customGroup } of externalApiItems) {
+        if (!externalItem) continue;
+
+        let { item, groups } = externalItem;
+    
+        // Make sure the new item doesn't have any conflicting IDs
+        baseId = incrementIds(item, baseId);
+
+        // Add the new item to the root children
+        typedocJson.children.push(item);
+
+        if (customGroup) {
+            groups = [customGroup];
+        } else if (groups.length === 0) {
+            groups = ['Reexports'];
+        }
+
+        for (const groupName of groups) {
+            // Assign the new item into the specified groups
+            const reexportsGroup = typedocJson.groups.find(g => g.title === groupName);
+    
+            if (reexportsGroup) {
+                reexportsGroup.children.push(item.id);
+            } else {
+                typedocJson.groups.push({ title: groupName, children: [item.id] });
+            }
+        }
+
+        console.log(`Reexported item ${item.name} from ${item.sources[0].fileName} to ${groups.join(', ')}`);
+    }
+
+    fs.writeFileSync(typedocJsonFilePath, JSON.stringify(typedocJson, null, 4));
+}