gitnexus 1.6.5-rc.7 → 1.6.5-rc.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -30,8 +30,11 @@ const readConfig = () => {
|
|
|
30
30
|
const rawDims = process.env.GITNEXUS_EMBEDDING_DIMS;
|
|
31
31
|
let dimensions;
|
|
32
32
|
if (rawDims !== undefined) {
|
|
33
|
+
if (!/^\d+$/.test(rawDims)) {
|
|
34
|
+
throw new Error(`GITNEXUS_EMBEDDING_DIMS must be a positive integer, got "${rawDims}"`);
|
|
35
|
+
}
|
|
33
36
|
const parsed = parseInt(rawDims, 10);
|
|
34
|
-
if (
|
|
37
|
+
if (parsed <= 0) {
|
|
35
38
|
throw new Error(`GITNEXUS_EMBEDDING_DIMS must be a positive integer, got "${rawDims}"`);
|
|
36
39
|
}
|
|
37
40
|
dimensions = parsed;
|
|
@@ -73,9 +76,22 @@ const safeUrl = (url) => {
|
|
|
73
76
|
* @param model - Model name for the request body
|
|
74
77
|
* @param apiKey - Bearer token (only used in Authorization header)
|
|
75
78
|
* @param batchIndex - Logical batch number (for error context)
|
|
76
|
-
* @param
|
|
79
|
+
* @param dimensions - Optional output-vector size. When provided, sent as
|
|
80
|
+
* the `dimensions` field in the request body. Endpoints that implement
|
|
81
|
+
* Matryoshka truncation (OpenAI text-embedding-3-*, Cohere embed-v3,
|
|
82
|
+
* Voyage) return a truncated vector at that size; endpoints that do not
|
|
83
|
+
* recognise the field may ignore it or return 400. Leave
|
|
84
|
+
* `GITNEXUS_EMBEDDING_DIMS` unset for strict backends that reject
|
|
85
|
+
* unknown fields.
|
|
77
86
|
*/
|
|
78
|
-
const httpEmbedBatch = async (url, batch, model, apiKey, batchIndex = 0) => {
|
|
87
|
+
const httpEmbedBatch = async (url, batch, model, apiKey, batchIndex = 0, dimensions) => {
|
|
88
|
+
const requestBody = {
|
|
89
|
+
input: batch,
|
|
90
|
+
model,
|
|
91
|
+
};
|
|
92
|
+
if (dimensions !== undefined) {
|
|
93
|
+
requestBody.dimensions = dimensions;
|
|
94
|
+
}
|
|
79
95
|
let resp;
|
|
80
96
|
try {
|
|
81
97
|
resp = await resilientFetch(url, {
|
|
@@ -85,7 +101,7 @@ const httpEmbedBatch = async (url, batch, model, apiKey, batchIndex = 0) => {
|
|
|
85
101
|
'Content-Type': 'application/json',
|
|
86
102
|
Authorization: `Bearer ${apiKey}`,
|
|
87
103
|
},
|
|
88
|
-
body: JSON.stringify(
|
|
104
|
+
body: JSON.stringify(requestBody),
|
|
89
105
|
}, {
|
|
90
106
|
breakerKey: HTTP_BREAKER_KEY,
|
|
91
107
|
retry: { maxAttempts: HTTP_MAX_RETRIES + 1, baseDelayMs: HTTP_RETRY_BACKOFF_MS },
|
|
@@ -130,7 +146,7 @@ export const httpEmbed = async (texts) => {
|
|
|
130
146
|
for (let i = 0; i < texts.length; i += HTTP_BATCH_SIZE) {
|
|
131
147
|
const batch = texts.slice(i, i + HTTP_BATCH_SIZE);
|
|
132
148
|
const batchIndex = Math.floor(i / HTTP_BATCH_SIZE);
|
|
133
|
-
const items = await httpEmbedBatch(url, batch, config.model, config.apiKey, batchIndex);
|
|
149
|
+
const items = await httpEmbedBatch(url, batch, config.model, config.apiKey, batchIndex, config.dimensions);
|
|
134
150
|
if (items.length !== batch.length) {
|
|
135
151
|
throw new Error(`Embedding endpoint returned ${items.length} vectors for ${batch.length} texts ` +
|
|
136
152
|
`(${safeUrl(url)}, batch ${batchIndex})`);
|
|
@@ -164,7 +180,7 @@ export const httpEmbedQuery = async (text) => {
|
|
|
164
180
|
if (!config)
|
|
165
181
|
throw new Error('HTTP embedding not configured');
|
|
166
182
|
const url = `${config.baseUrl}/embeddings`;
|
|
167
|
-
const items = await httpEmbedBatch(url, [text], config.model, config.apiKey);
|
|
183
|
+
const items = await httpEmbedBatch(url, [text], config.model, config.apiKey, 0, config.dimensions);
|
|
168
184
|
if (!items.length) {
|
|
169
185
|
throw new Error(`Embedding endpoint returned empty response (${safeUrl(url)})`);
|
|
170
186
|
}
|
|
@@ -5,6 +5,7 @@ import { compilePatterns, runCompiledPatterns, unquoteLiteral, } from '../tree-s
|
|
|
5
5
|
* - FastAPI `@app.get("/path")` provider decorators
|
|
6
6
|
* - `requests.get/post/...("url")` consumer calls
|
|
7
7
|
* - Generic `requests.request("METHOD", "url")` consumer calls
|
|
8
|
+
* - `httpx.AsyncClient` instances calling `.get/.post/...("url")`
|
|
8
9
|
*/
|
|
9
10
|
const FASTAPI_VERBS = {
|
|
10
11
|
get: 'GET',
|
|
@@ -65,11 +66,142 @@ const REQUESTS_GENERIC_PATTERNS = compilePatterns({
|
|
|
65
66
|
},
|
|
66
67
|
],
|
|
67
68
|
});
|
|
69
|
+
// ─── Consumer: httpx.AsyncClient assignments ────────────────────────
|
|
70
|
+
// NOTE: This targeted detector only tracks explicit `httpx.AsyncClient(...)`
|
|
71
|
+
// construction. Direct imports (`from httpx import AsyncClient`) and module
|
|
72
|
+
// aliases (`import httpx as hx`) and annotated assignments (`client: httpx.AsyncClient = ...`)
|
|
73
|
+
// are intentionally left for a follow-up. Module-scope clients are only matched
|
|
74
|
+
// at module scope; calls inside functions require a function/class-local tracked
|
|
75
|
+
// client to avoid false positives from same-name local variables.
|
|
76
|
+
const HTTPX_ASYNC_CLIENT_ASSIGN_PATTERNS = compilePatterns({
|
|
77
|
+
name: 'python-httpx-async-client-assign',
|
|
78
|
+
language: Python,
|
|
79
|
+
patterns: [
|
|
80
|
+
{
|
|
81
|
+
meta: {},
|
|
82
|
+
query: `
|
|
83
|
+
(assignment
|
|
84
|
+
left: (_) @client
|
|
85
|
+
right: (call
|
|
86
|
+
function: (attribute
|
|
87
|
+
object: (identifier) @module (#eq? @module "httpx")
|
|
88
|
+
attribute: (identifier) @client_class (#eq? @client_class "AsyncClient"))))
|
|
89
|
+
`,
|
|
90
|
+
},
|
|
91
|
+
],
|
|
92
|
+
});
|
|
93
|
+
// ─── Consumer: async with httpx.AsyncClient() as client ──────────────
|
|
94
|
+
const HTTPX_ASYNC_CLIENT_WITH_ALIAS_PATTERNS = compilePatterns({
|
|
95
|
+
name: 'python-httpx-async-client-with-alias',
|
|
96
|
+
language: Python,
|
|
97
|
+
patterns: [
|
|
98
|
+
{
|
|
99
|
+
meta: {},
|
|
100
|
+
query: `
|
|
101
|
+
(as_pattern
|
|
102
|
+
(call
|
|
103
|
+
function: (attribute
|
|
104
|
+
object: (identifier) @module (#eq? @module "httpx")
|
|
105
|
+
attribute: (identifier) @client_class (#eq? @client_class "AsyncClient")))
|
|
106
|
+
(as_pattern_target (identifier) @client))
|
|
107
|
+
`,
|
|
108
|
+
},
|
|
109
|
+
],
|
|
110
|
+
});
|
|
111
|
+
function getScopeKey(node, preferClass = false) {
|
|
112
|
+
if (preferClass) {
|
|
113
|
+
let current = node;
|
|
114
|
+
while (current) {
|
|
115
|
+
if (current.type === 'class_definition') {
|
|
116
|
+
return `class:${current.startIndex}:${current.endIndex}`;
|
|
117
|
+
}
|
|
118
|
+
current = current.parent;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
let current = node;
|
|
122
|
+
while (current) {
|
|
123
|
+
if (current.type === 'function_definition') {
|
|
124
|
+
return `function:${current.startIndex}:${current.endIndex}`;
|
|
125
|
+
}
|
|
126
|
+
current = current.parent;
|
|
127
|
+
}
|
|
128
|
+
return 'module';
|
|
129
|
+
}
|
|
130
|
+
function trackedClientScopeKey(clientNode) {
|
|
131
|
+
return getScopeKey(clientNode.parent, clientNode.text.includes('.'));
|
|
132
|
+
}
|
|
133
|
+
function callScopeKeys(clientNode) {
|
|
134
|
+
const keys = new Set();
|
|
135
|
+
const preferClass = clientNode.text.includes('.');
|
|
136
|
+
const nearestScope = getScopeKey(clientNode.parent, preferClass);
|
|
137
|
+
keys.add(nearestScope);
|
|
138
|
+
return [...keys];
|
|
139
|
+
}
|
|
140
|
+
function collectHttpxAsyncClients(tree) {
|
|
141
|
+
const clients = new Map();
|
|
142
|
+
const addClient = (clientNode) => {
|
|
143
|
+
if (!clientNode)
|
|
144
|
+
return;
|
|
145
|
+
const scopeKey = trackedClientScopeKey(clientNode);
|
|
146
|
+
const clientText = clientNode.text;
|
|
147
|
+
const scopes = clients.get(clientText) ?? new Set();
|
|
148
|
+
scopes.add(scopeKey);
|
|
149
|
+
clients.set(clientText, scopes);
|
|
150
|
+
};
|
|
151
|
+
for (const match of runCompiledPatterns(HTTPX_ASYNC_CLIENT_ASSIGN_PATTERNS, tree)) {
|
|
152
|
+
addClient(match.captures.client);
|
|
153
|
+
}
|
|
154
|
+
for (const match of runCompiledPatterns(HTTPX_ASYNC_CLIENT_WITH_ALIAS_PATTERNS, tree)) {
|
|
155
|
+
addClient(match.captures.client);
|
|
156
|
+
}
|
|
157
|
+
return clients;
|
|
158
|
+
}
|
|
159
|
+
function hasTrackedHttpxAsyncClient(clients, clientNode) {
|
|
160
|
+
const scopes = clients.get(clientNode.text);
|
|
161
|
+
if (!scopes)
|
|
162
|
+
return false;
|
|
163
|
+
return callScopeKeys(clientNode).some((scopeKey) => scopes.has(scopeKey));
|
|
164
|
+
}
|
|
165
|
+
// ─── Consumer: httpx AsyncClient .get/.post/...("url") ──────────────
|
|
166
|
+
const HTTPX_ASYNC_CLIENT_VERB_PATTERNS = compilePatterns({
|
|
167
|
+
name: 'python-httpx-async-client-verb',
|
|
168
|
+
language: Python,
|
|
169
|
+
patterns: [
|
|
170
|
+
{
|
|
171
|
+
meta: {},
|
|
172
|
+
query: `
|
|
173
|
+
(call
|
|
174
|
+
function: (attribute
|
|
175
|
+
object: (_) @client
|
|
176
|
+
attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
|
|
177
|
+
arguments: (argument_list . (string) @path))
|
|
178
|
+
`,
|
|
179
|
+
},
|
|
180
|
+
],
|
|
181
|
+
});
|
|
182
|
+
// ─── Consumer: httpx AsyncClient .request("METHOD", "url") ─────────
|
|
183
|
+
const HTTPX_ASYNC_CLIENT_GENERIC_PATTERNS = compilePatterns({
|
|
184
|
+
name: 'python-httpx-async-client-generic',
|
|
185
|
+
language: Python,
|
|
186
|
+
patterns: [
|
|
187
|
+
{
|
|
188
|
+
meta: {},
|
|
189
|
+
query: `
|
|
190
|
+
(call
|
|
191
|
+
function: (attribute
|
|
192
|
+
object: (_) @client
|
|
193
|
+
attribute: (identifier) @method (#eq? @method "request"))
|
|
194
|
+
arguments: (argument_list . (string) @http_method (string) @path))
|
|
195
|
+
`,
|
|
196
|
+
},
|
|
197
|
+
],
|
|
198
|
+
});
|
|
68
199
|
export const PYTHON_HTTP_PLUGIN = {
|
|
69
200
|
name: 'python-http',
|
|
70
201
|
language: Python,
|
|
71
202
|
scan(tree) {
|
|
72
203
|
const out = [];
|
|
204
|
+
const httpxAsyncClients = collectHttpxAsyncClients(tree);
|
|
73
205
|
// Providers: FastAPI
|
|
74
206
|
for (const match of runCompiledPatterns(FASTAPI_PATTERNS, tree)) {
|
|
75
207
|
const methodNode = match.captures.method;
|
|
@@ -128,6 +260,49 @@ export const PYTHON_HTTP_PLUGIN = {
|
|
|
128
260
|
confidence: 0.7,
|
|
129
261
|
});
|
|
130
262
|
}
|
|
263
|
+
// Consumers: httpx.AsyncClient.<verb>("url")
|
|
264
|
+
for (const match of runCompiledPatterns(HTTPX_ASYNC_CLIENT_VERB_PATTERNS, tree)) {
|
|
265
|
+
const clientNode = match.captures.client;
|
|
266
|
+
const methodNode = match.captures.method;
|
|
267
|
+
const pathNode = match.captures.path;
|
|
268
|
+
if (!clientNode || !methodNode || !pathNode)
|
|
269
|
+
continue;
|
|
270
|
+
if (!hasTrackedHttpxAsyncClient(httpxAsyncClients, clientNode))
|
|
271
|
+
continue;
|
|
272
|
+
const path = unquoteLiteral(pathNode.text);
|
|
273
|
+
if (path === null)
|
|
274
|
+
continue;
|
|
275
|
+
out.push({
|
|
276
|
+
role: 'consumer',
|
|
277
|
+
framework: 'python-httpx',
|
|
278
|
+
method: methodNode.text.toUpperCase(),
|
|
279
|
+
path,
|
|
280
|
+
name: null,
|
|
281
|
+
confidence: 0.7,
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
// Consumers: httpx.AsyncClient.request("METHOD", "url")
|
|
285
|
+
for (const match of runCompiledPatterns(HTTPX_ASYNC_CLIENT_GENERIC_PATTERNS, tree)) {
|
|
286
|
+
const clientNode = match.captures.client;
|
|
287
|
+
const methodNode = match.captures.http_method;
|
|
288
|
+
const pathNode = match.captures.path;
|
|
289
|
+
if (!clientNode || !methodNode || !pathNode)
|
|
290
|
+
continue;
|
|
291
|
+
if (!hasTrackedHttpxAsyncClient(httpxAsyncClients, clientNode))
|
|
292
|
+
continue;
|
|
293
|
+
const methodRaw = unquoteLiteral(methodNode.text);
|
|
294
|
+
const path = unquoteLiteral(pathNode.text);
|
|
295
|
+
if (methodRaw === null || path === null)
|
|
296
|
+
continue;
|
|
297
|
+
out.push({
|
|
298
|
+
role: 'consumer',
|
|
299
|
+
framework: 'python-httpx',
|
|
300
|
+
method: methodRaw.toUpperCase(),
|
|
301
|
+
path,
|
|
302
|
+
name: null,
|
|
303
|
+
confidence: 0.7,
|
|
304
|
+
});
|
|
305
|
+
}
|
|
131
306
|
return out;
|
|
132
307
|
},
|
|
133
308
|
};
|
package/package.json
CHANGED