@yangfei_93sky/biocli 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +197 -0
- package/dist/batch.d.ts +20 -0
- package/dist/batch.js +69 -0
- package/dist/build-manifest.d.ts +38 -0
- package/dist/build-manifest.js +186 -0
- package/dist/cache.d.ts +28 -0
- package/dist/cache.js +126 -0
- package/dist/cli-manifest.json +1500 -0
- package/dist/cli.d.ts +7 -0
- package/dist/cli.js +336 -0
- package/dist/clis/_shared/common.d.ts +8 -0
- package/dist/clis/_shared/common.js +13 -0
- package/dist/clis/_shared/eutils.d.ts +9 -0
- package/dist/clis/_shared/eutils.js +9 -0
- package/dist/clis/_shared/organism-db.d.ts +23 -0
- package/dist/clis/_shared/organism-db.js +58 -0
- package/dist/clis/_shared/xml-helpers.d.ts +58 -0
- package/dist/clis/_shared/xml-helpers.js +266 -0
- package/dist/clis/aggregate/enrichment.d.ts +7 -0
- package/dist/clis/aggregate/enrichment.js +105 -0
- package/dist/clis/aggregate/gene-dossier.d.ts +13 -0
- package/dist/clis/aggregate/gene-dossier.js +248 -0
- package/dist/clis/aggregate/gene-profile.d.ts +16 -0
- package/dist/clis/aggregate/gene-profile.js +305 -0
- package/dist/clis/aggregate/literature-brief.d.ts +7 -0
- package/dist/clis/aggregate/literature-brief.js +79 -0
- package/dist/clis/aggregate/variant-dossier.d.ts +11 -0
- package/dist/clis/aggregate/variant-dossier.js +161 -0
- package/dist/clis/aggregate/variant-interpret.d.ts +10 -0
- package/dist/clis/aggregate/variant-interpret.js +210 -0
- package/dist/clis/aggregate/workflow-prepare.d.ts +12 -0
- package/dist/clis/aggregate/workflow-prepare.js +228 -0
- package/dist/clis/aggregate/workflow-scout.d.ts +13 -0
- package/dist/clis/aggregate/workflow-scout.js +175 -0
- package/dist/clis/clinvar/search.d.ts +8 -0
- package/dist/clis/clinvar/search.js +61 -0
- package/dist/clis/clinvar/variant.d.ts +7 -0
- package/dist/clis/clinvar/variant.js +53 -0
- package/dist/clis/enrichr/analyze.d.ts +7 -0
- package/dist/clis/enrichr/analyze.js +48 -0
- package/dist/clis/ensembl/lookup.d.ts +6 -0
- package/dist/clis/ensembl/lookup.js +38 -0
- package/dist/clis/ensembl/vep.d.ts +7 -0
- package/dist/clis/ensembl/vep.js +86 -0
- package/dist/clis/ensembl/xrefs.d.ts +6 -0
- package/dist/clis/ensembl/xrefs.js +36 -0
- package/dist/clis/gene/fetch.d.ts +10 -0
- package/dist/clis/gene/fetch.js +96 -0
- package/dist/clis/gene/info.d.ts +7 -0
- package/dist/clis/gene/info.js +37 -0
- package/dist/clis/gene/search.d.ts +7 -0
- package/dist/clis/gene/search.js +71 -0
- package/dist/clis/geo/dataset.d.ts +7 -0
- package/dist/clis/geo/dataset.js +55 -0
- package/dist/clis/geo/download.d.ts +17 -0
- package/dist/clis/geo/download.js +115 -0
- package/dist/clis/geo/samples.d.ts +7 -0
- package/dist/clis/geo/samples.js +57 -0
- package/dist/clis/geo/search.d.ts +8 -0
- package/dist/clis/geo/search.js +66 -0
- package/dist/clis/kegg/convert.d.ts +7 -0
- package/dist/clis/kegg/convert.js +37 -0
- package/dist/clis/kegg/disease.d.ts +6 -0
- package/dist/clis/kegg/disease.js +57 -0
- package/dist/clis/kegg/link.d.ts +7 -0
- package/dist/clis/kegg/link.js +36 -0
- package/dist/clis/kegg/pathway.d.ts +6 -0
- package/dist/clis/kegg/pathway.js +37 -0
- package/dist/clis/pubmed/abstract.d.ts +7 -0
- package/dist/clis/pubmed/abstract.js +42 -0
- package/dist/clis/pubmed/cited-by.d.ts +7 -0
- package/dist/clis/pubmed/cited-by.js +77 -0
- package/dist/clis/pubmed/fetch.d.ts +6 -0
- package/dist/clis/pubmed/fetch.js +36 -0
- package/dist/clis/pubmed/info.yaml +22 -0
- package/dist/clis/pubmed/related.d.ts +7 -0
- package/dist/clis/pubmed/related.js +81 -0
- package/dist/clis/pubmed/search.d.ts +8 -0
- package/dist/clis/pubmed/search.js +63 -0
- package/dist/clis/snp/lookup.d.ts +7 -0
- package/dist/clis/snp/lookup.js +57 -0
- package/dist/clis/sra/download.d.ts +18 -0
- package/dist/clis/sra/download.js +217 -0
- package/dist/clis/sra/run.d.ts +8 -0
- package/dist/clis/sra/run.js +77 -0
- package/dist/clis/sra/search.d.ts +8 -0
- package/dist/clis/sra/search.js +83 -0
- package/dist/clis/string/enrichment.d.ts +7 -0
- package/dist/clis/string/enrichment.js +50 -0
- package/dist/clis/string/network.d.ts +7 -0
- package/dist/clis/string/network.js +47 -0
- package/dist/clis/string/partners.d.ts +4 -0
- package/dist/clis/string/partners.js +44 -0
- package/dist/clis/taxonomy/lookup.d.ts +8 -0
- package/dist/clis/taxonomy/lookup.js +54 -0
- package/dist/clis/uniprot/fetch.d.ts +7 -0
- package/dist/clis/uniprot/fetch.js +82 -0
- package/dist/clis/uniprot/search.d.ts +6 -0
- package/dist/clis/uniprot/search.js +65 -0
- package/dist/clis/uniprot/sequence.d.ts +7 -0
- package/dist/clis/uniprot/sequence.js +51 -0
- package/dist/commander-adapter.d.ts +27 -0
- package/dist/commander-adapter.js +286 -0
- package/dist/completion.d.ts +19 -0
- package/dist/completion.js +117 -0
- package/dist/config.d.ts +57 -0
- package/dist/config.js +94 -0
- package/dist/databases/enrichr.d.ts +28 -0
- package/dist/databases/enrichr.js +131 -0
- package/dist/databases/ensembl.d.ts +14 -0
- package/dist/databases/ensembl.js +106 -0
- package/dist/databases/index.d.ts +45 -0
- package/dist/databases/index.js +49 -0
- package/dist/databases/kegg.d.ts +26 -0
- package/dist/databases/kegg.js +136 -0
- package/dist/databases/ncbi.d.ts +28 -0
- package/dist/databases/ncbi.js +144 -0
- package/dist/databases/string-db.d.ts +19 -0
- package/dist/databases/string-db.js +105 -0
- package/dist/databases/uniprot.d.ts +13 -0
- package/dist/databases/uniprot.js +110 -0
- package/dist/discovery.d.ts +32 -0
- package/dist/discovery.js +235 -0
- package/dist/doctor.d.ts +19 -0
- package/dist/doctor.js +151 -0
- package/dist/errors.d.ts +68 -0
- package/dist/errors.js +105 -0
- package/dist/execution.d.ts +15 -0
- package/dist/execution.js +178 -0
- package/dist/hooks.d.ts +48 -0
- package/dist/hooks.js +58 -0
- package/dist/main.d.ts +13 -0
- package/dist/main.js +31 -0
- package/dist/ncbi-fetch.d.ts +10 -0
- package/dist/ncbi-fetch.js +10 -0
- package/dist/output.d.ts +18 -0
- package/dist/output.js +394 -0
- package/dist/pipeline/executor.d.ts +22 -0
- package/dist/pipeline/executor.js +40 -0
- package/dist/pipeline/index.d.ts +6 -0
- package/dist/pipeline/index.js +6 -0
- package/dist/pipeline/registry.d.ts +16 -0
- package/dist/pipeline/registry.js +31 -0
- package/dist/pipeline/steps/fetch.d.ts +21 -0
- package/dist/pipeline/steps/fetch.js +160 -0
- package/dist/pipeline/steps/transform.d.ts +26 -0
- package/dist/pipeline/steps/transform.js +92 -0
- package/dist/pipeline/steps/xml-parse.d.ts +12 -0
- package/dist/pipeline/steps/xml-parse.js +27 -0
- package/dist/pipeline/template.d.ts +35 -0
- package/dist/pipeline/template.js +312 -0
- package/dist/rate-limiter.d.ts +56 -0
- package/dist/rate-limiter.js +120 -0
- package/dist/registry-api.d.ts +15 -0
- package/dist/registry-api.js +13 -0
- package/dist/registry.d.ts +90 -0
- package/dist/registry.js +100 -0
- package/dist/schema.d.ts +80 -0
- package/dist/schema.js +72 -0
- package/dist/spinner.d.ts +19 -0
- package/dist/spinner.js +37 -0
- package/dist/types.d.ts +101 -0
- package/dist/types.js +27 -0
- package/dist/utils.d.ts +16 -0
- package/dist/utils.js +40 -0
- package/dist/validate.d.ts +29 -0
- package/dist/validate.js +136 -0
- package/dist/verify.d.ts +20 -0
- package/dist/verify.js +131 -0
- package/dist/version.d.ts +13 -0
- package/dist/version.js +36 -0
- package/dist/xml-parser.d.ts +19 -0
- package/dist/xml-parser.js +119 -0
- package/dist/yaml-schema.d.ts +40 -0
- package/dist/yaml-schema.js +62 -0
- package/package.json +68 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline step: fetch — HTTP API requests for NCBI endpoints.
|
|
3
|
+
*
|
|
4
|
+
* Adapted from opencli:
|
|
5
|
+
* - No browser-related fetch paths (no page.evaluate, no fetchBatchInBrowser)
|
|
6
|
+
* - Always uses Node.js built-in fetch()
|
|
7
|
+
* - Automatic API key / email injection via HttpContext
|
|
8
|
+
* - Rate limiting integration via HttpContext.fetch() or standalone rate limiter
|
|
9
|
+
* - XML auto-detection: if Content-Type contains 'xml', auto-parse with parseXml()
|
|
10
|
+
* - Per-item fetch pattern (when data is array and URL contains `item`)
|
|
11
|
+
* - Configurable concurrency (default 5)
|
|
12
|
+
*/
|
|
13
|
+
import { CliError, getErrorMessage } from '../../errors.js';
|
|
14
|
+
import { renderTemplate } from '../template.js';
|
|
15
|
+
import { isRecord, mapConcurrent } from '../../utils.js';
|
|
16
|
+
/**
|
|
17
|
+
* Check if a Content-Type header indicates XML.
|
|
18
|
+
*/
|
|
19
|
+
function isXmlContentType(contentType) {
|
|
20
|
+
if (!contentType)
|
|
21
|
+
return false;
|
|
22
|
+
return contentType.includes('xml');
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Parse an XML string into a JS object using fast-xml-parser.
|
|
26
|
+
* Lazily imports to avoid top-level dependency issues.
|
|
27
|
+
*/
|
|
28
|
+
async function parseXmlResponse(text) {
|
|
29
|
+
const { XMLParser } = await import('fast-xml-parser');
|
|
30
|
+
const parser = new XMLParser({
|
|
31
|
+
ignoreAttributes: false,
|
|
32
|
+
attributeNamePrefix: '@_',
|
|
33
|
+
textNodeName: '#text',
|
|
34
|
+
isArray: (_name, _jpath, isLeafNode, isAttribute) => {
|
|
35
|
+
// Keep leaf text nodes as scalars; arrays only for repeated elements
|
|
36
|
+
if (isAttribute || isLeafNode)
|
|
37
|
+
return false;
|
|
38
|
+
return false;
|
|
39
|
+
},
|
|
40
|
+
});
|
|
41
|
+
return parser.parse(text);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Build a URL with query parameters appended.
|
|
45
|
+
*/
|
|
46
|
+
function appendQueryParams(url, params) {
|
|
47
|
+
if (Object.keys(params).length === 0)
|
|
48
|
+
return url;
|
|
49
|
+
const qs = new URLSearchParams(params).toString();
|
|
50
|
+
return `${url}${url.includes('?') ? '&' : '?'}${qs}`;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Inject NCBI API key and email into query params if available on the context.
|
|
54
|
+
*/
|
|
55
|
+
function injectNcbiParams(params, ctx) {
|
|
56
|
+
const result = { ...params };
|
|
57
|
+
if (ctx?.apiKey && !result.api_key) {
|
|
58
|
+
result.api_key = ctx.apiKey;
|
|
59
|
+
}
|
|
60
|
+
if (ctx?.email && !result.email) {
|
|
61
|
+
result.email = ctx.email;
|
|
62
|
+
}
|
|
63
|
+
return result;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Perform a single HTTP fetch with optional rate limiting via HttpContext.
|
|
67
|
+
* Auto-detects XML responses and parses them.
|
|
68
|
+
*/
|
|
69
|
+
async function fetchSingle(ctx, url, method, queryParams, headers) {
|
|
70
|
+
// Inject NCBI credentials
|
|
71
|
+
const mergedParams = injectNcbiParams(queryParams, ctx);
|
|
72
|
+
const finalUrl = appendQueryParams(url, mergedParams);
|
|
73
|
+
let resp;
|
|
74
|
+
if (ctx) {
|
|
75
|
+
// Use context's rate-limited fetch
|
|
76
|
+
resp = await ctx.fetch(finalUrl, {
|
|
77
|
+
method: method.toUpperCase(),
|
|
78
|
+
headers,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
// Direct fetch (no rate limiting)
|
|
83
|
+
resp = await fetch(finalUrl, {
|
|
84
|
+
method: method.toUpperCase(),
|
|
85
|
+
headers,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
if (!resp.ok) {
|
|
89
|
+
throw new CliError('FETCH_ERROR', `HTTP ${resp.status} ${resp.statusText} from ${finalUrl}`);
|
|
90
|
+
}
|
|
91
|
+
// Auto-detect XML and parse accordingly
|
|
92
|
+
const contentType = resp.headers.get('content-type');
|
|
93
|
+
const text = await resp.text();
|
|
94
|
+
if (isXmlContentType(contentType)) {
|
|
95
|
+
return parseXmlResponse(text);
|
|
96
|
+
}
|
|
97
|
+
// Try JSON
|
|
98
|
+
try {
|
|
99
|
+
return JSON.parse(text);
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
// Return raw text if not parseable as JSON
|
|
103
|
+
return text;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Pipeline fetch step handler.
|
|
108
|
+
*
|
|
109
|
+
* Params can be:
|
|
110
|
+
* - A string (URL template)
|
|
111
|
+
* - An object with: url, method, params, headers, concurrency
|
|
112
|
+
*/
|
|
113
|
+
export async function handleFetch(ctx, params, data, args) {
|
|
114
|
+
const paramObject = isRecord(params) ? params : {};
|
|
115
|
+
const urlOrObj = typeof params === 'string' ? params : (paramObject.url ?? '');
|
|
116
|
+
const method = typeof paramObject.method === 'string' ? paramObject.method : 'GET';
|
|
117
|
+
const queryParams = isRecord(paramObject.params)
|
|
118
|
+
? paramObject.params
|
|
119
|
+
: {};
|
|
120
|
+
const headers = isRecord(paramObject.headers)
|
|
121
|
+
? paramObject.headers
|
|
122
|
+
: {};
|
|
123
|
+
const urlTemplate = String(urlOrObj);
|
|
124
|
+
// Per-item fetch when data is array and URL references item
|
|
125
|
+
if (Array.isArray(data) && urlTemplate.includes('item')) {
|
|
126
|
+
const concurrency = typeof paramObject.concurrency === 'number' ? paramObject.concurrency : 5;
|
|
127
|
+
// Render headers and query params once (they don't depend on item)
|
|
128
|
+
const renderedHeaders = {};
|
|
129
|
+
for (const [k, v] of Object.entries(headers)) {
|
|
130
|
+
renderedHeaders[k] = String(renderTemplate(v, { args, data }));
|
|
131
|
+
}
|
|
132
|
+
const renderedQueryParams = {};
|
|
133
|
+
for (const [k, v] of Object.entries(queryParams)) {
|
|
134
|
+
renderedQueryParams[k] = String(renderTemplate(v, { args, data }));
|
|
135
|
+
}
|
|
136
|
+
// Fetch each item concurrently with bounded concurrency
|
|
137
|
+
return mapConcurrent(data, async (item, index) => {
|
|
138
|
+
const itemUrl = String(renderTemplate(urlTemplate, { args, data, item, index }));
|
|
139
|
+
try {
|
|
140
|
+
return await fetchSingle(ctx, itemUrl, method, renderedQueryParams, renderedHeaders);
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
const message = getErrorMessage(error);
|
|
144
|
+
return { error: message };
|
|
145
|
+
}
|
|
146
|
+
}, concurrency);
|
|
147
|
+
}
|
|
148
|
+
// Single fetch
|
|
149
|
+
const url = String(renderTemplate(urlOrObj, { args, data }));
|
|
150
|
+
// Render query params and headers with current context
|
|
151
|
+
const renderedQueryParams = {};
|
|
152
|
+
for (const [k, v] of Object.entries(queryParams)) {
|
|
153
|
+
renderedQueryParams[k] = String(renderTemplate(v, { args, data }));
|
|
154
|
+
}
|
|
155
|
+
const renderedHeaders = {};
|
|
156
|
+
for (const [k, v] of Object.entries(headers)) {
|
|
157
|
+
renderedHeaders[k] = String(renderTemplate(v, { args, data }));
|
|
158
|
+
}
|
|
159
|
+
return fetchSingle(ctx, url, method, renderedQueryParams, renderedHeaders);
|
|
160
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline steps: data transforms — select, map, filter, sort, limit.
|
|
3
|
+
*/
|
|
4
|
+
import type { HttpContext } from '../../types.js';
|
|
5
|
+
/**
|
|
6
|
+
* Navigate a nested path (e.g. `esearchresult.idlist`) within the current data.
|
|
7
|
+
*/
|
|
8
|
+
export declare function handleSelect(_ctx: HttpContext | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
|
|
9
|
+
/**
|
|
10
|
+
* Transform array items using template expressions.
|
|
11
|
+
* Supports inline select via `{ map: { select: 'path', key: '${{ item.x }}' } }`.
|
|
12
|
+
*/
|
|
13
|
+
export declare function handleMap(_ctx: HttpContext | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
|
|
14
|
+
/**
|
|
15
|
+
* Filter array items by a template expression that evaluates to truthy/falsy.
|
|
16
|
+
*/
|
|
17
|
+
export declare function handleFilter(_ctx: HttpContext | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
|
|
18
|
+
/**
|
|
19
|
+
* Sort array items by a field, ascending or descending.
|
|
20
|
+
* Params can be a string (field name) or `{ by: 'field', order: 'desc' }`.
|
|
21
|
+
*/
|
|
22
|
+
export declare function handleSort(_ctx: HttpContext | null, params: unknown, data: unknown, _args: Record<string, unknown>): Promise<unknown>;
|
|
23
|
+
/**
|
|
24
|
+
* Truncate an array to the first N items.
|
|
25
|
+
*/
|
|
26
|
+
export declare function handleLimit(_ctx: HttpContext | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline steps: data transforms — select, map, filter, sort, limit.
|
|
3
|
+
*/
|
|
4
|
+
import { renderTemplate, evalExpr } from '../template.js';
|
|
5
|
+
import { isRecord } from '../../utils.js';
|
|
6
|
+
/**
|
|
7
|
+
* Navigate a nested path (e.g. `esearchresult.idlist`) within the current data.
|
|
8
|
+
*/
|
|
9
|
+
export async function handleSelect(_ctx, params, data, args) {
|
|
10
|
+
const pathStr = String(renderTemplate(params, { args, data }));
|
|
11
|
+
if (data && typeof data === 'object') {
|
|
12
|
+
let current = data;
|
|
13
|
+
for (const part of pathStr.split('.')) {
|
|
14
|
+
if (isRecord(current)) {
|
|
15
|
+
current = current[part];
|
|
16
|
+
}
|
|
17
|
+
else if (Array.isArray(current) && /^\d+$/.test(part)) {
|
|
18
|
+
current = current[parseInt(part, 10)];
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return current;
|
|
25
|
+
}
|
|
26
|
+
return data;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Transform array items using template expressions.
|
|
30
|
+
* Supports inline select via `{ map: { select: 'path', key: '${{ item.x }}' } }`.
|
|
31
|
+
*/
|
|
32
|
+
export async function handleMap(_ctx, params, data, args) {
|
|
33
|
+
if (!data || typeof data !== 'object')
|
|
34
|
+
return data;
|
|
35
|
+
let source = data;
|
|
36
|
+
// Support inline select: { map: { select: 'path', key: '${{ item.x }}' } }
|
|
37
|
+
if (isRecord(params) && 'select' in params) {
|
|
38
|
+
source = await handleSelect(null, params.select, data, args);
|
|
39
|
+
}
|
|
40
|
+
if (!source || typeof source !== 'object')
|
|
41
|
+
return source;
|
|
42
|
+
let items = Array.isArray(source) ? source : [source];
|
|
43
|
+
if (isRecord(source) && Array.isArray(source.data))
|
|
44
|
+
items = source.data;
|
|
45
|
+
const result = [];
|
|
46
|
+
const templateParams = isRecord(params) ? params : {};
|
|
47
|
+
for (let i = 0; i < items.length; i++) {
|
|
48
|
+
const item = items[i];
|
|
49
|
+
const row = {};
|
|
50
|
+
for (const [key, template] of Object.entries(templateParams)) {
|
|
51
|
+
if (key === 'select')
|
|
52
|
+
continue;
|
|
53
|
+
row[key] = renderTemplate(template, { args, data: source, item, index: i });
|
|
54
|
+
}
|
|
55
|
+
result.push(row);
|
|
56
|
+
}
|
|
57
|
+
return result;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Filter array items by a template expression that evaluates to truthy/falsy.
|
|
61
|
+
*/
|
|
62
|
+
export async function handleFilter(_ctx, params, data, args) {
|
|
63
|
+
if (!Array.isArray(data))
|
|
64
|
+
return data;
|
|
65
|
+
return data.filter((item, i) => evalExpr(String(params), { args, item, index: i }));
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Sort array items by a field, ascending or descending.
|
|
69
|
+
* Params can be a string (field name) or `{ by: 'field', order: 'desc' }`.
|
|
70
|
+
*/
|
|
71
|
+
export async function handleSort(_ctx, params, data, _args) {
|
|
72
|
+
if (!Array.isArray(data))
|
|
73
|
+
return data;
|
|
74
|
+
const key = isRecord(params) ? String(params.by ?? '') : String(params);
|
|
75
|
+
const reverse = isRecord(params) ? params.order === 'desc' : false;
|
|
76
|
+
return [...data].sort((a, b) => {
|
|
77
|
+
const left = isRecord(a) ? a[key] : undefined;
|
|
78
|
+
const right = isRecord(b) ? b[key] : undefined;
|
|
79
|
+
const cmp = String(left ?? '').localeCompare(String(right ?? ''), undefined, {
|
|
80
|
+
numeric: true,
|
|
81
|
+
});
|
|
82
|
+
return reverse ? -cmp : cmp;
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Truncate an array to the first N items.
|
|
87
|
+
*/
|
|
88
|
+
export async function handleLimit(_ctx, params, data, args) {
|
|
89
|
+
if (!Array.isArray(data))
|
|
90
|
+
return data;
|
|
91
|
+
return data.slice(0, Number(renderTemplate(params, { args, data })));
|
|
92
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline step: xml-parse — parse XML strings into JSON objects.
|
|
3
|
+
*
|
|
4
|
+
* Uses fast-xml-parser (same as the fetch step's auto-detection path)
|
|
5
|
+
* for explicit XML parsing when the data hasn't been auto-parsed yet.
|
|
6
|
+
*/
|
|
7
|
+
import type { HttpContext } from '../../types.js';
|
|
8
|
+
/**
|
|
9
|
+
* Parse an XML string into a JavaScript object.
|
|
10
|
+
* If data is already parsed (not a string), returns it unchanged.
|
|
11
|
+
*/
|
|
12
|
+
export declare function handleXmlParse(_ctx: HttpContext | null, _params: unknown, data: unknown, _args: Record<string, unknown>): Promise<unknown>;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline step: xml-parse — parse XML strings into JSON objects.
|
|
3
|
+
*
|
|
4
|
+
* Uses fast-xml-parser (same as the fetch step's auto-detection path)
|
|
5
|
+
* for explicit XML parsing when the data hasn't been auto-parsed yet.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Parse an XML string into a JavaScript object.
|
|
9
|
+
* If data is already parsed (not a string), returns it unchanged.
|
|
10
|
+
*/
|
|
11
|
+
export async function handleXmlParse(_ctx, _params, data, _args) {
|
|
12
|
+
if (typeof data === 'string') {
|
|
13
|
+
const { XMLParser } = await import('fast-xml-parser');
|
|
14
|
+
const parser = new XMLParser({
|
|
15
|
+
ignoreAttributes: false,
|
|
16
|
+
attributeNamePrefix: '@_',
|
|
17
|
+
textNodeName: '#text',
|
|
18
|
+
isArray: (_name, _jpath, isLeafNode, isAttribute) => {
|
|
19
|
+
if (isAttribute || isLeafNode)
|
|
20
|
+
return false;
|
|
21
|
+
return false;
|
|
22
|
+
},
|
|
23
|
+
});
|
|
24
|
+
return parser.parse(data);
|
|
25
|
+
}
|
|
26
|
+
return data; // already parsed
|
|
27
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline template engine: ${{ ... }} expression rendering.
|
|
3
|
+
*
|
|
4
|
+
* Ported from opencli with all features intact:
|
|
5
|
+
* - ${{ expr }} syntax with interpolation
|
|
6
|
+
* - Dot-path resolution (fast path)
|
|
7
|
+
* - String/numeric literals (fast path)
|
|
8
|
+
* - Pipe filters: upper, lower, trim, truncate(n), replace(old,new),
|
|
9
|
+
* join(sep), first, last, length, keys, json, default(val),
|
|
10
|
+
* slugify, sanitize, urlencode, urldecode, ext, basename
|
|
11
|
+
* - VM-sandboxed JavaScript expressions fallback
|
|
12
|
+
* - LRU cache for compiled scripts
|
|
13
|
+
* - Security: forbidden pattern blocking
|
|
14
|
+
*/
|
|
15
|
+
export interface RenderContext {
|
|
16
|
+
args?: Record<string, unknown>;
|
|
17
|
+
data?: unknown;
|
|
18
|
+
item?: unknown;
|
|
19
|
+
index?: number;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Render a template string (or pass through non-strings unchanged).
|
|
23
|
+
*
|
|
24
|
+
* - If the entire string is a single `${{ ... }}`, the raw evaluated value
|
|
25
|
+
* is returned (preserving type — number, array, object, etc.).
|
|
26
|
+
* - Otherwise embedded expressions are stringified and interpolated.
|
|
27
|
+
*/
|
|
28
|
+
export declare function renderTemplate(template: unknown, ctx: RenderContext): unknown;
|
|
29
|
+
/**
|
|
30
|
+
* Recursively render all template strings inside a value (object / array / string).
|
|
31
|
+
* Non-string primitives are returned as-is.
|
|
32
|
+
*/
|
|
33
|
+
export declare function renderValue(value: unknown, ctx: RenderContext): unknown;
|
|
34
|
+
export declare function evalExpr(expr: string, ctx: RenderContext): unknown;
|
|
35
|
+
export declare function resolvePath(pathStr: string, ctx: RenderContext): unknown;
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline template engine: ${{ ... }} expression rendering.
|
|
3
|
+
*
|
|
4
|
+
* Ported from opencli with all features intact:
|
|
5
|
+
* - ${{ expr }} syntax with interpolation
|
|
6
|
+
* - Dot-path resolution (fast path)
|
|
7
|
+
* - String/numeric literals (fast path)
|
|
8
|
+
* - Pipe filters: upper, lower, trim, truncate(n), replace(old,new),
|
|
9
|
+
* join(sep), first, last, length, keys, json, default(val),
|
|
10
|
+
* slugify, sanitize, urlencode, urldecode, ext, basename
|
|
11
|
+
* - VM-sandboxed JavaScript expressions fallback
|
|
12
|
+
* - LRU cache for compiled scripts
|
|
13
|
+
* - Security: forbidden pattern blocking
|
|
14
|
+
*/
|
|
15
|
+
import vm from 'node:vm';
|
|
16
|
+
import { isRecord } from '../utils.js';
|
|
17
|
+
/**
|
|
18
|
+
* Render a template string (or pass through non-strings unchanged).
|
|
19
|
+
*
|
|
20
|
+
* - If the entire string is a single `${{ ... }}`, the raw evaluated value
|
|
21
|
+
* is returned (preserving type — number, array, object, etc.).
|
|
22
|
+
* - Otherwise embedded expressions are stringified and interpolated.
|
|
23
|
+
*/
|
|
24
|
+
export function renderTemplate(template, ctx) {
|
|
25
|
+
if (typeof template !== 'string')
|
|
26
|
+
return template;
|
|
27
|
+
const trimmed = template.trim();
|
|
28
|
+
// Full expression: entire string is a single ${{ ... }}
|
|
29
|
+
// Use [^}] to prevent matching across }} boundaries (e.g. "${{ a }}-${{ b }}")
|
|
30
|
+
const fullMatch = trimmed.match(/^\$\{\{\s*([^}]*(?:\}[^}][^}]*)*)\s*\}\}$/);
|
|
31
|
+
if (fullMatch && !trimmed.includes('}}-') && !trimmed.includes('}}${{')) {
|
|
32
|
+
return evalExpr(fullMatch[1].trim(), ctx);
|
|
33
|
+
}
|
|
34
|
+
// Check if the entire string is a single expression (no other text around it)
|
|
35
|
+
const singleExpr = trimmed.match(/^\$\{\{\s*([\s\S]*?)\s*\}\}$/);
|
|
36
|
+
if (singleExpr) {
|
|
37
|
+
// Verify it's truly a single expression (no other ${{ inside)
|
|
38
|
+
const inner = singleExpr[1];
|
|
39
|
+
if (!inner.includes('${{'))
|
|
40
|
+
return evalExpr(inner.trim(), ctx);
|
|
41
|
+
}
|
|
42
|
+
return template.replace(/\$\{\{\s*(.*?)\s*\}\}/g, (_m, expr) => String(evalExpr(expr.trim(), ctx)));
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Recursively render all template strings inside a value (object / array / string).
|
|
46
|
+
* Non-string primitives are returned as-is.
|
|
47
|
+
*/
|
|
48
|
+
export function renderValue(value, ctx) {
|
|
49
|
+
if (typeof value === 'string')
|
|
50
|
+
return renderTemplate(value, ctx);
|
|
51
|
+
if (Array.isArray(value))
|
|
52
|
+
return value.map((v) => renderValue(v, ctx));
|
|
53
|
+
if (isRecord(value)) {
|
|
54
|
+
const out = {};
|
|
55
|
+
for (const [k, v] of Object.entries(value)) {
|
|
56
|
+
out[k] = renderValue(v, ctx);
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
60
|
+
return value;
|
|
61
|
+
}
|
|
62
|
+
// ── Expression evaluator ────────────────────────────────────────────────────
|
|
63
|
+
export function evalExpr(expr, ctx) {
|
|
64
|
+
const args = ctx.args ?? {};
|
|
65
|
+
const item = ctx.item ?? {};
|
|
66
|
+
const data = ctx.data;
|
|
67
|
+
const index = ctx.index ?? 0;
|
|
68
|
+
// ── Pipe filters: expr | filter1(arg) | filter2 ──
|
|
69
|
+
// Split on single | (not ||) so "item.a || item.b | upper" works correctly.
|
|
70
|
+
const pipeSegments = expr.split(/(?<!\|)\|(?!\|)/).map((s) => s.trim());
|
|
71
|
+
if (pipeSegments.length > 1) {
|
|
72
|
+
let result = evalExpr(pipeSegments[0], ctx);
|
|
73
|
+
for (let i = 1; i < pipeSegments.length; i++) {
|
|
74
|
+
result = applyFilter(pipeSegments[i], result);
|
|
75
|
+
}
|
|
76
|
+
return result;
|
|
77
|
+
}
|
|
78
|
+
// Fast path: quoted string literal — skip VM overhead
|
|
79
|
+
const strLit = expr.match(/^(['"])(.*)\1$/);
|
|
80
|
+
if (strLit)
|
|
81
|
+
return strLit[2];
|
|
82
|
+
// Fast path: numeric literal
|
|
83
|
+
if (/^\d+(\.\d+)?$/.test(expr))
|
|
84
|
+
return Number(expr);
|
|
85
|
+
// Try resolving as a simple dotted path (item.foo.bar, args.limit, index)
|
|
86
|
+
const resolved = resolvePath(expr, { args, item, data, index });
|
|
87
|
+
if (resolved !== null && resolved !== undefined)
|
|
88
|
+
return resolved;
|
|
89
|
+
// Fallback: evaluate as JS in a sandboxed VM.
|
|
90
|
+
// Handles ||, ??, arithmetic, ternary, method calls, etc. natively.
|
|
91
|
+
return evalJsExpr(expr, { args, item, data, index });
|
|
92
|
+
}
|
|
93
|
+
// ── Pipe filters ────────────────────────────────────────────────────────────
|
|
94
|
+
/**
|
|
95
|
+
* Apply a named filter to a value.
|
|
96
|
+
* Supported filters:
|
|
97
|
+
* default(val), join(sep), upper, lower, truncate(n), trim,
|
|
98
|
+
* replace(old,new), keys, length, first, last, json,
|
|
99
|
+
* slugify, sanitize, urlencode, urldecode, ext, basename
|
|
100
|
+
*/
|
|
101
|
+
function applyFilter(filterExpr, value) {
|
|
102
|
+
const match = filterExpr.match(/^(\w+)(?:\((.+)\))?$/);
|
|
103
|
+
if (!match)
|
|
104
|
+
return value;
|
|
105
|
+
const [, name, rawArgs] = match;
|
|
106
|
+
const filterArg = rawArgs?.replace(/^['"]|['"]$/g, '') ?? '';
|
|
107
|
+
switch (name) {
|
|
108
|
+
case 'default': {
|
|
109
|
+
if (value === null || value === undefined || value === '') {
|
|
110
|
+
const intVal = parseInt(filterArg, 10);
|
|
111
|
+
if (!Number.isNaN(intVal) && String(intVal) === filterArg.trim())
|
|
112
|
+
return intVal;
|
|
113
|
+
return filterArg;
|
|
114
|
+
}
|
|
115
|
+
return value;
|
|
116
|
+
}
|
|
117
|
+
case 'join':
|
|
118
|
+
return Array.isArray(value) ? value.join(filterArg || ', ') : value;
|
|
119
|
+
case 'upper':
|
|
120
|
+
return typeof value === 'string' ? value.toUpperCase() : value;
|
|
121
|
+
case 'lower':
|
|
122
|
+
return typeof value === 'string' ? value.toLowerCase() : value;
|
|
123
|
+
case 'trim':
|
|
124
|
+
return typeof value === 'string' ? value.trim() : value;
|
|
125
|
+
case 'truncate': {
|
|
126
|
+
const n = parseInt(filterArg, 10) || 50;
|
|
127
|
+
return typeof value === 'string' && value.length > n
|
|
128
|
+
? `${value.slice(0, n)}...`
|
|
129
|
+
: value;
|
|
130
|
+
}
|
|
131
|
+
case 'replace': {
|
|
132
|
+
if (typeof value !== 'string')
|
|
133
|
+
return value;
|
|
134
|
+
const parts = rawArgs?.split(',').map((s) => s.trim().replace(/^['"]|['"]$/g, '')) ?? [];
|
|
135
|
+
return parts.length >= 2 ? value.replaceAll(parts[0], parts[1]) : value;
|
|
136
|
+
}
|
|
137
|
+
case 'keys':
|
|
138
|
+
return value && typeof value === 'object' ? Object.keys(value) : value;
|
|
139
|
+
case 'length':
|
|
140
|
+
return Array.isArray(value)
|
|
141
|
+
? value.length
|
|
142
|
+
: typeof value === 'string'
|
|
143
|
+
? value.length
|
|
144
|
+
: value;
|
|
145
|
+
case 'first':
|
|
146
|
+
return Array.isArray(value) ? value[0] : value;
|
|
147
|
+
case 'last':
|
|
148
|
+
return Array.isArray(value) ? value[value.length - 1] : value;
|
|
149
|
+
case 'json':
|
|
150
|
+
return JSON.stringify(value ?? null);
|
|
151
|
+
case 'slugify':
|
|
152
|
+
// Convert to URL-safe slug
|
|
153
|
+
return typeof value === 'string'
|
|
154
|
+
? value
|
|
155
|
+
.toLowerCase()
|
|
156
|
+
.replace(/[^\p{L}\p{N}]+/gu, '-')
|
|
157
|
+
.replace(/^-|-$/g, '')
|
|
158
|
+
: value;
|
|
159
|
+
case 'sanitize':
|
|
160
|
+
// Remove invalid filename characters
|
|
161
|
+
return typeof value === 'string'
|
|
162
|
+
? // biome-ignore lint/suspicious/noControlCharactersInRegex: intentional - strips C0 control chars from filenames
|
|
163
|
+
value.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_')
|
|
164
|
+
: value;
|
|
165
|
+
case 'ext': {
|
|
166
|
+
// Extract file extension from URL or path
|
|
167
|
+
if (typeof value !== 'string')
|
|
168
|
+
return value;
|
|
169
|
+
const lastDot = value.lastIndexOf('.');
|
|
170
|
+
const lastSlash = Math.max(value.lastIndexOf('/'), value.lastIndexOf('\\'));
|
|
171
|
+
return lastDot > lastSlash ? value.slice(lastDot) : '';
|
|
172
|
+
}
|
|
173
|
+
case 'basename': {
|
|
174
|
+
// Extract filename from URL or path
|
|
175
|
+
if (typeof value !== 'string')
|
|
176
|
+
return value;
|
|
177
|
+
const parts = value.split(/[/\\]/);
|
|
178
|
+
return parts[parts.length - 1] || value;
|
|
179
|
+
}
|
|
180
|
+
case 'urlencode':
|
|
181
|
+
return typeof value === 'string' ? encodeURIComponent(value) : value;
|
|
182
|
+
case 'urldecode':
|
|
183
|
+
return typeof value === 'string' ? decodeURIComponent(value) : value;
|
|
184
|
+
default:
|
|
185
|
+
return value;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
// ── Dot-path resolution ─────────────────────────────────────────────────────
|
|
189
|
+
export function resolvePath(pathStr, ctx) {
|
|
190
|
+
const args = ctx.args ?? {};
|
|
191
|
+
const item = ctx.item ?? {};
|
|
192
|
+
const data = ctx.data;
|
|
193
|
+
const index = ctx.index ?? 0;
|
|
194
|
+
const parts = pathStr.split('.');
|
|
195
|
+
const rootName = parts[0];
|
|
196
|
+
let obj;
|
|
197
|
+
let rest;
|
|
198
|
+
if (rootName === 'args') {
|
|
199
|
+
obj = args;
|
|
200
|
+
rest = parts.slice(1);
|
|
201
|
+
}
|
|
202
|
+
else if (rootName === 'item') {
|
|
203
|
+
obj = item;
|
|
204
|
+
rest = parts.slice(1);
|
|
205
|
+
}
|
|
206
|
+
else if (rootName === 'data') {
|
|
207
|
+
obj = data;
|
|
208
|
+
rest = parts.slice(1);
|
|
209
|
+
}
|
|
210
|
+
else if (rootName === 'index') {
|
|
211
|
+
return index;
|
|
212
|
+
}
|
|
213
|
+
else {
|
|
214
|
+
// Default: treat as item property
|
|
215
|
+
obj = item;
|
|
216
|
+
rest = parts;
|
|
217
|
+
}
|
|
218
|
+
for (const part of rest) {
|
|
219
|
+
if (isRecord(obj)) {
|
|
220
|
+
obj = obj[part];
|
|
221
|
+
}
|
|
222
|
+
else if (Array.isArray(obj) && /^\d+$/.test(part)) {
|
|
223
|
+
obj = obj[parseInt(part, 10)];
|
|
224
|
+
}
|
|
225
|
+
else {
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return obj;
|
|
230
|
+
}
|
|
231
|
+
// ── Sandboxed JS evaluation ─────────────────────────────────────────────────
|
|
232
|
+
/**
|
|
233
|
+
* Evaluate arbitrary JS expressions as a last-resort fallback.
|
|
234
|
+
* Runs inside a `node:vm` sandbox with dynamic code generation disabled.
|
|
235
|
+
*
|
|
236
|
+
* Compiled functions are cached by expression string to avoid re-creating
|
|
237
|
+
* VM contexts on every invocation — critical for loops where the same
|
|
238
|
+
* expression is evaluated hundreds of times.
|
|
239
|
+
*/
|
|
240
|
+
const FORBIDDEN_EXPR_PATTERNS = /\b(constructor|__proto__|prototype|globalThis|process|require|import|eval)\b/;
|
|
241
|
+
/**
|
|
242
|
+
* Deep-copy plain data to sever prototype chains, preventing sandbox escape
|
|
243
|
+
* via `args.constructor.constructor('return process')()` etc.
|
|
244
|
+
*/
|
|
245
|
+
function sanitizeContext(obj) {
|
|
246
|
+
if (obj === null || obj === undefined)
|
|
247
|
+
return obj;
|
|
248
|
+
if (typeof obj !== 'object' && typeof obj !== 'function')
|
|
249
|
+
return obj;
|
|
250
|
+
try {
|
|
251
|
+
return JSON.parse(JSON.stringify(obj));
|
|
252
|
+
}
|
|
253
|
+
catch {
|
|
254
|
+
return {};
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
/** LRU-bounded cache for compiled VM scripts — prevents unbounded memory growth. */
|
|
258
|
+
const MAX_VM_CACHE_SIZE = 256;
|
|
259
|
+
const _vmCache = new Map();
|
|
260
|
+
function getOrCompileScript(expr) {
|
|
261
|
+
let script = _vmCache.get(expr);
|
|
262
|
+
if (script)
|
|
263
|
+
return script;
|
|
264
|
+
// Evict oldest entry when cache is full
|
|
265
|
+
if (_vmCache.size >= MAX_VM_CACHE_SIZE) {
|
|
266
|
+
const firstKey = _vmCache.keys().next().value;
|
|
267
|
+
if (firstKey !== undefined)
|
|
268
|
+
_vmCache.delete(firstKey);
|
|
269
|
+
}
|
|
270
|
+
script = new vm.Script(`(${expr})`);
|
|
271
|
+
_vmCache.set(expr, script);
|
|
272
|
+
return script;
|
|
273
|
+
}
|
|
274
|
+
function evalJsExpr(expr, ctx) {
|
|
275
|
+
// Guard against absurdly long expressions that could indicate injection.
|
|
276
|
+
if (expr.length > 2000)
|
|
277
|
+
return undefined;
|
|
278
|
+
// Block obvious sandbox escape attempts.
|
|
279
|
+
if (FORBIDDEN_EXPR_PATTERNS.test(expr))
|
|
280
|
+
return undefined;
|
|
281
|
+
const args = sanitizeContext(ctx.args ?? {});
|
|
282
|
+
const item = sanitizeContext(ctx.item ?? {});
|
|
283
|
+
const data = sanitizeContext(ctx.data);
|
|
284
|
+
const index = ctx.index ?? 0;
|
|
285
|
+
try {
|
|
286
|
+
const script = getOrCompileScript(expr);
|
|
287
|
+
const sandbox = vm.createContext({
|
|
288
|
+
args,
|
|
289
|
+
item,
|
|
290
|
+
data,
|
|
291
|
+
index,
|
|
292
|
+
encodeURIComponent,
|
|
293
|
+
decodeURIComponent,
|
|
294
|
+
JSON,
|
|
295
|
+
Math,
|
|
296
|
+
Number,
|
|
297
|
+
String,
|
|
298
|
+
Boolean,
|
|
299
|
+
Array,
|
|
300
|
+
Date,
|
|
301
|
+
}, {
|
|
302
|
+
codeGeneration: {
|
|
303
|
+
strings: false,
|
|
304
|
+
wasm: false,
|
|
305
|
+
},
|
|
306
|
+
});
|
|
307
|
+
return script.runInContext(sandbox, { timeout: 50 });
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
return undefined;
|
|
311
|
+
}
|
|
312
|
+
}
|