hlquery-node-client 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +37 -0
- package/LICENSE +29 -0
- package/README.md +316 -0
- package/example.js +636 -0
- package/examples/basic_usage.js +35 -0
- package/examples/collections.js +40 -0
- package/examples/csv.js +39 -0
- package/examples/documents.js +54 -0
- package/examples/flush.js +128 -0
- package/examples/keys.js +58 -0
- package/examples/pdf.js +40 -0
- package/examples/search.js +101 -0
- package/examples/vector.js +465 -0
- package/index.js +39 -0
- package/lib/Aliases.js +48 -0
- package/lib/Client.js +623 -0
- package/lib/Collections.js +159 -0
- package/lib/Documents.js +244 -0
- package/lib/Exceptions.js +101 -0
- package/lib/Keys.js +85 -0
- package/lib/Overrides.js +53 -0
- package/lib/Request.js +147 -0
- package/lib/Response.js +54 -0
- package/lib/Search.js +350 -0
- package/lib/Stopwords.js +60 -0
- package/lib/Synonyms.js +85 -0
- package/lib/System.js +110 -0
- package/lib/conformance.js +74 -0
- package/package.json +42 -0
- package/test/offline-smoke.js +86 -0
- package/utils/Auth.js +34 -0
- package/utils/CSVParser.js +160 -0
- package/utils/Config.js +57 -0
- package/utils/PDFParser.js +196 -0
- package/utils/Validator.js +143 -0
- package/utils/ranker.js +51 -0
package/lib/System.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hlquery Node.js Client - System API
|
|
3
|
+
*
|
|
4
|
+
* Copyright (C) 2021-2026, Carlos F. Ferry <carlos.ferry@gmail.com>
|
|
5
|
+
*
|
|
6
|
+
* This file is part of hlquery, released under the BSD License version 3.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* System and operational API operations
|
|
11
|
+
*/
|
|
12
|
+
class System {
|
|
13
|
+
constructor(request) {
|
|
14
|
+
this.request = request;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
async health() {
|
|
18
|
+
return await this.request.execute('GET', '/health');
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async status() {
|
|
22
|
+
return await this.request.execute('GET', '/status');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async startup() {
|
|
26
|
+
return await this.request.execute('GET', '/startup');
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async bootStatus() {
|
|
30
|
+
return await this.request.execute('GET', '/boot-status');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async info() {
|
|
34
|
+
return await this.request.execute('GET', '/');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async stats() {
|
|
38
|
+
return await this.request.execute('GET', '/stats');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async metrics() {
|
|
42
|
+
return await this.request.execute('GET', '/metrics');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async metricsJson() {
|
|
46
|
+
return await this.request.execute('GET', '/metrics.json');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async connections() {
|
|
50
|
+
return await this.request.execute('GET', '/connections');
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async rocksdb() {
|
|
54
|
+
return await this.request.execute('GET', '/rocksdb');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async rocksdbInternal() {
|
|
58
|
+
return await this.request.execute('GET', '/_rocksdb');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async docTotal() {
|
|
62
|
+
return await this.request.execute('GET', '/doctotal');
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async etc() {
|
|
66
|
+
return await this.request.execute('GET', '/etc');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async sql(sql, params = {}) {
|
|
70
|
+
if (typeof sql !== 'string' || sql.trim() === '') {
|
|
71
|
+
throw new Error('SQL query must be a non-empty string');
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
if (params === null || typeof params !== 'object' || Array.isArray(params)) {
|
|
75
|
+
throw new Error('SQL params must be an object');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return await this.request.execute('GET', '/sql', null, { ...params, sql });
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async execSql(sql) {
|
|
82
|
+
if (typeof sql !== 'string' || sql.trim() === '') {
|
|
83
|
+
throw new Error('SQL query must be a non-empty string');
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return await this.request.execute('POST', '/sql', { exec: sql });
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async ping() {
|
|
90
|
+
return await this.request.execute('GET', '/ping');
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async integrity() {
|
|
94
|
+
return await this.request.execute('GET', '/integrity');
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async consistency() {
|
|
98
|
+
return await this.request.execute('GET', '/consistency');
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async selfCheck() {
|
|
102
|
+
return await this.request.execute('GET', '/self-check');
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async storageStatus() {
|
|
106
|
+
return await this.request.execute('GET', '/admin/storage_status');
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
module.exports = System;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hlquery Node.js Client - Server conformance manifest
|
|
3
|
+
*
|
|
4
|
+
* This manifest maps routed server endpoints to the Node client wrapper that
|
|
5
|
+
* owns them. Any server route in scope must be marked as supported or omitted.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const NODE_CLIENT_ROUTE_COVERAGE = [
|
|
9
|
+
{ path: '/', methods: ['GET'], status: 'supported', client: 'system.info' },
|
|
10
|
+
{ path: '/health', methods: ['GET'], status: 'supported', client: 'system.health' },
|
|
11
|
+
{ path: '/status', methods: ['GET'], status: 'supported', client: 'system.status' },
|
|
12
|
+
{ path: '/startup', methods: ['GET'], status: 'supported', client: 'system.startup' },
|
|
13
|
+
{ path: '/boot-status', methods: ['GET'], status: 'supported', client: 'system.bootStatus' },
|
|
14
|
+
{ path: '/stats', methods: ['GET'], status: 'supported', client: 'system.stats' },
|
|
15
|
+
{ path: '/metrics', methods: ['GET'], status: 'supported', client: 'system.metrics' },
|
|
16
|
+
{ path: '/metrics.json', methods: ['GET'], status: 'supported', client: 'system.metricsJson' },
|
|
17
|
+
{ path: '/connections', methods: ['GET'], status: 'supported', client: 'system.connections' },
|
|
18
|
+
{ path: '/rocksdb', methods: ['GET'], status: 'supported', client: 'system.rocksdb' },
|
|
19
|
+
{ path: '/_rocksdb', methods: ['GET'], status: 'supported', client: 'system.rocksdbInternal' },
|
|
20
|
+
{ path: '/doctotal', methods: ['GET'], status: 'supported', client: 'system.docTotal' },
|
|
21
|
+
{ path: '/etc', methods: ['GET'], status: 'supported', client: 'system.etc' },
|
|
22
|
+
{ path: '/ping', methods: ['GET'], status: 'supported', client: 'system.ping' },
|
|
23
|
+
{ path: '/integrity', methods: ['GET'], status: 'supported', client: 'system.integrity' },
|
|
24
|
+
{ path: '/consistency', methods: ['GET'], status: 'supported', client: 'system.consistency' },
|
|
25
|
+
{ path: '/self-check', methods: ['GET'], status: 'supported', client: 'system.selfCheck' },
|
|
26
|
+
{ path: '/admin/storage_status', methods: ['GET'], status: 'supported', client: 'system.storageStatus' },
|
|
27
|
+
{ path: '/cluster/health', methods: ['GET'], status: 'supported', client: 'client.clusterHealth' },
|
|
28
|
+
{ path: '/cluster/stats', methods: ['GET'], status: 'supported', client: 'client.clusterStats' },
|
|
29
|
+
{ path: '/cluster/nodes', methods: ['GET'], status: 'supported', client: 'client.clusterNodes' },
|
|
30
|
+
{ path: '/links', methods: ['GET'], status: 'supported', client: 'client.links' },
|
|
31
|
+
{ path: '/links/ping', methods: ['GET'], status: 'supported', client: 'client.linksPing' },
|
|
32
|
+
{ path: '/links/connect', methods: ['POST'], status: 'supported', client: 'client.linksConnect' },
|
|
33
|
+
{ path: '/links/disconnect', methods: ['POST'], status: 'supported', client: 'client.linksDisconnect' },
|
|
34
|
+
{ path: '/flush', methods: ['POST'], status: 'supported', client: 'client.flush' },
|
|
35
|
+
{ path: '/collections/distributed', methods: ['GET'], status: 'supported', client: 'client.listCollectionsDistributed' },
|
|
36
|
+
{ path: '/collections', methods: ['GET', 'POST'], status: 'supported', client: 'collections.list/create' },
|
|
37
|
+
{ path: '/collections/{name}', methods: ['GET', 'DELETE'], status: 'supported', client: 'collections.get/delete' },
|
|
38
|
+
{ path: '/collections/{name}/update', methods: ['POST'], status: 'supported', client: 'collections.update' },
|
|
39
|
+
{ path: '/collections/{name}/documents', methods: ['GET', 'POST', 'DELETE'], status: 'supported', client: 'documents.list/add/deleteByFilter' },
|
|
40
|
+
{ path: '/collections/{name}/documents/import', methods: ['POST'], status: 'supported', client: 'documents.import' },
|
|
41
|
+
{ path: '/collections/{name}/documents/{id}', methods: ['GET', 'PUT', 'DELETE'], status: 'supported', client: 'documents.get/update/delete' },
|
|
42
|
+
{ path: '/collections/{name}/search', methods: ['GET', 'POST'], status: 'supported', client: 'search.search' },
|
|
43
|
+
{ path: '/collections/{name}/documents/search', methods: ['GET', 'POST'], status: 'supported', client: 'search.searchLegacy' },
|
|
44
|
+
{ path: '/collections/{name}/vector_search', methods: ['GET', 'POST'], status: 'supported', client: 'search.vectorSearch' },
|
|
45
|
+
{ path: '/multi_search', methods: ['GET', 'POST'], status: 'supported', client: 'search.multiSearch' },
|
|
46
|
+
{ path: '/search', methods: ['GET', 'POST'], status: 'supported', client: 'search.globalSearch' },
|
|
47
|
+
{ path: '/collections/{name}/documents/facet_counts', methods: ['GET', 'POST'], status: 'supported', client: 'documents.facetCounts' },
|
|
48
|
+
{ path: '/collections/{name}/documents/export', methods: ['GET', 'POST'], status: 'supported', client: 'documents.export' },
|
|
49
|
+
{ path: '/collections/{name}/synonyms', methods: ['GET'], status: 'supported', client: 'synonyms.list' },
|
|
50
|
+
{ path: '/collections/{name}/synonyms/{id}', methods: ['GET', 'POST', 'PUT', 'DELETE'], status: 'supported', client: 'synonyms.get/create/update/delete' },
|
|
51
|
+
{ path: '/synonyms', methods: ['GET'], status: 'supported', client: 'synonyms.listAll' },
|
|
52
|
+
{ path: '/synonyms/global', methods: ['GET'], status: 'supported', client: 'synonyms.listGlobal' },
|
|
53
|
+
{ path: '/synonyms/global/{id}', methods: ['GET', 'POST', 'PUT', 'DELETE'], status: 'supported', client: 'synonyms.getGlobal/createGlobal/updateGlobal/deleteGlobal' },
|
|
54
|
+
{ path: '/collections/{name}/stopwords', methods: ['GET', 'POST'], status: 'supported', client: 'stopwords.list/create' },
|
|
55
|
+
{ path: '/collections/{name}/stopwords/{word}', methods: ['DELETE'], status: 'supported', client: 'stopwords.delete' },
|
|
56
|
+
{ path: '/stopwords', methods: ['GET'], status: 'supported', client: 'stopwords.listAll' },
|
|
57
|
+
{ path: '/stopwords/global', methods: ['GET', 'POST'], status: 'supported', client: 'stopwords.listGlobal/createGlobal' },
|
|
58
|
+
{ path: '/stopwords/global/{word}', methods: ['DELETE'], status: 'supported', client: 'stopwords.deleteGlobal' },
|
|
59
|
+
{ path: '/collections/{name}/overrides', methods: ['GET'], status: 'supported', client: 'overrides.list' },
|
|
60
|
+
{ path: '/collections/{name}/overrides/{id}', methods: ['GET', 'POST', 'PUT', 'DELETE'], status: 'supported', client: 'overrides.get/create/update/delete' },
|
|
61
|
+
{ path: '/aliases', methods: ['GET'], status: 'supported', client: 'aliases.list' },
|
|
62
|
+
{ path: '/aliases/{name}', methods: ['GET', 'POST', 'PUT', 'DELETE'], status: 'supported', client: 'aliases.get/create/update/delete' },
|
|
63
|
+
{ path: '/keys', methods: ['GET', 'POST'], status: 'supported', client: 'keys.list/create' },
|
|
64
|
+
{ path: '/keys/{id}', methods: ['GET', 'PUT', 'DELETE'], status: 'supported', client: 'keys.get/update/delete' },
|
|
65
|
+
{ path: '/update-counters', methods: ['GET', 'POST'], status: 'omitted', reason: 'Internal maintenance endpoint; intentionally left off the public Node client.' },
|
|
66
|
+
{ path: '/query', methods: ['GET'], status: 'omitted', reason: 'Alias of /status; client exposes status() instead of duplicate naming.' },
|
|
67
|
+
{ path: '/modules', methods: ['GET'], status: 'omitted', reason: 'Module APIs are intentionally omitted until the module surface is documented and stabilized for SDKs.' },
|
|
68
|
+
{ path: '/modules/{name}', methods: ['ANY'], status: 'omitted', reason: 'Module-specific routes are dynamic and intentionally not wrapped in the stable Node client.' },
|
|
69
|
+
{ path: '/modules/{name}/syntax', methods: ['GET'], status: 'omitted', reason: 'Module-specific routes are dynamic and intentionally not wrapped in the stable Node client.' }
|
|
70
|
+
];
|
|
71
|
+
|
|
72
|
+
module.exports = {
|
|
73
|
+
NODE_CLIENT_ROUTE_COVERAGE
|
|
74
|
+
};
|
package/package.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "hlquery-node-client",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Node.js client library for hlquery search engine",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"type": "commonjs",
|
|
7
|
+
"scripts": {
|
|
8
|
+
"test": "node ../tests/test_node_client_conformance.js",
|
|
9
|
+
"test:conformance": "node ../tests/test_node_client_conformance.js",
|
|
10
|
+
"test:offline": "node test/offline-smoke.js",
|
|
11
|
+
"example": "node example.js",
|
|
12
|
+
"example:pdf": "node examples/pdf.js",
|
|
13
|
+
"example:csv": "node examples/csv.js"
|
|
14
|
+
},
|
|
15
|
+
"dependencies": {
|
|
16
|
+
"pdf-parse": "^2.4.5"
|
|
17
|
+
},
|
|
18
|
+
"keywords": [
|
|
19
|
+
"hlquery",
|
|
20
|
+
"search",
|
|
21
|
+
"client",
|
|
22
|
+
"api"
|
|
23
|
+
],
|
|
24
|
+
"author": "Carlos F. Ferry <carlos.ferry@gmail.com>",
|
|
25
|
+
"license": "BSD-3-Clause",
|
|
26
|
+
"engines": {
|
|
27
|
+
"node": ">=12.0.0"
|
|
28
|
+
},
|
|
29
|
+
"repository": {
|
|
30
|
+
"type": "git",
|
|
31
|
+
"url": "git+https://github.com/cferry/hlquery.git"
|
|
32
|
+
},
|
|
33
|
+
"directories": {
|
|
34
|
+
"example": "examples",
|
|
35
|
+
"lib": "lib",
|
|
36
|
+
"test": "test"
|
|
37
|
+
},
|
|
38
|
+
"bugs": {
|
|
39
|
+
"url": "https://github.com/cferry/hlquery/issues"
|
|
40
|
+
},
|
|
41
|
+
"homepage": "https://github.com/cferry/hlquery#readme"
|
|
42
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
const assert = require('assert');
|
|
4
|
+
const http = require('http');
|
|
5
|
+
|
|
6
|
+
const Hlquery = require('..');
|
|
7
|
+
const Client = require('../lib/Client');
|
|
8
|
+
const Request = require('../lib/Request');
|
|
9
|
+
const Response = require('../lib/Response');
|
|
10
|
+
const Config = require('../utils/Config');
|
|
11
|
+
const Validator = require('../utils/Validator');
|
|
12
|
+
|
|
13
|
+
async function main() {
|
|
14
|
+
assert.strictEqual(typeof Hlquery, 'function', 'default export should be the Client constructor');
|
|
15
|
+
assert.strictEqual(Hlquery.Client, Client, 'named Client export should match default export');
|
|
16
|
+
|
|
17
|
+
assert.strictEqual(Config.normalizeUrl('localhost:9200/'), 'http://localhost:9200');
|
|
18
|
+
assert.strictEqual(Config.isValidUrl('http://localhost:9200'), true);
|
|
19
|
+
assert.strictEqual(Config.isValidUrl('notaurl'), false);
|
|
20
|
+
|
|
21
|
+
Validator.validateCollectionName('books');
|
|
22
|
+
Validator.validateDocumentId('doc_123');
|
|
23
|
+
Validator.validateSearchParams({ limit: 10, offset: 0 });
|
|
24
|
+
|
|
25
|
+
assert.throws(() => Validator.validateCollectionName('123bad'), /letter or underscore/);
|
|
26
|
+
assert.throws(() => Validator.validateDocumentFields({ title: 'bad,value' }), /comma/);
|
|
27
|
+
|
|
28
|
+
const response = new Response(200, { ok: true }, { 'content-type': 'application/json' });
|
|
29
|
+
assert.strictEqual(response.isSuccess(), true);
|
|
30
|
+
assert.deepStrictEqual(response.getBody(), { ok: true });
|
|
31
|
+
|
|
32
|
+
const server = http.createServer((req, res) => {
|
|
33
|
+
if (req.url === '/health') {
|
|
34
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
35
|
+
res.end(JSON.stringify({ status: 'ok' }));
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (req.url === '/echo-auth') {
|
|
40
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
41
|
+
res.end(JSON.stringify({
|
|
42
|
+
authorization: req.headers.authorization || null,
|
|
43
|
+
apiKey: req.headers['x-api-key'] || null
|
|
44
|
+
}));
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
res.writeHead(404, { 'Content-Type': 'application/json' });
|
|
49
|
+
res.end(JSON.stringify({ error: 'not found' }));
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
await new Promise((resolve, reject) => {
|
|
53
|
+
server.listen(0, '127.0.0.1', (err) => (err ? reject(err) : resolve()));
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
const { port } = server.address();
|
|
57
|
+
|
|
58
|
+
try {
|
|
59
|
+
const request = new Request(`http://127.0.0.1:${port}`);
|
|
60
|
+
const health = await request.execute('GET', '/health');
|
|
61
|
+
assert.strictEqual(health.getStatusCode(), 200);
|
|
62
|
+
assert.deepStrictEqual(health.getBody(), { status: 'ok' });
|
|
63
|
+
|
|
64
|
+
request.setAuthToken('secret-token', 'bearer');
|
|
65
|
+
const bearerEcho = await request.execute('GET', '/echo-auth');
|
|
66
|
+
assert.strictEqual(bearerEcho.getBody().authorization, 'Bearer secret-token');
|
|
67
|
+
|
|
68
|
+
request.setAuthToken('scoped-key', 'api-key');
|
|
69
|
+
const apiKeyEcho = await request.execute('GET', '/echo-auth');
|
|
70
|
+
assert.strictEqual(apiKeyEcho.getBody().apiKey, 'scoped-key');
|
|
71
|
+
|
|
72
|
+
const client = new Client(`http://127.0.0.1:${port}`);
|
|
73
|
+
const clientHealth = await client.health();
|
|
74
|
+
assert.strictEqual(clientHealth.isSuccess(), true);
|
|
75
|
+
assert.deepStrictEqual(clientHealth.getBody(), { status: 'ok' });
|
|
76
|
+
} finally {
|
|
77
|
+
await new Promise((resolve) => server.close(resolve));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
process.stdout.write('Node offline smoke tests passed.\n');
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
main().catch((error) => {
|
|
84
|
+
process.stderr.write(`${error.stack || error.message}\n`);
|
|
85
|
+
process.exit(1);
|
|
86
|
+
});
|
package/utils/Auth.js
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hlquery Node.js Client - Authentication Utilities
|
|
3
|
+
*
|
|
4
|
+
* Copyright (C) 2021-2026, Carlos F. Ferry <carlos.ferry@gmail.com>
|
|
5
|
+
*
|
|
6
|
+
* This file is part of hlquery, released under the BSD License version 3.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const crypto = require('crypto');
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Authentication utilities
|
|
13
|
+
*/
|
|
14
|
+
class Auth {
|
|
15
|
+
/**
|
|
16
|
+
* Generate MD5 hash for a token (utility function for token generation)
|
|
17
|
+
* Note: Authentication only requires a token - no username/password needed
|
|
18
|
+
*/
|
|
19
|
+
static generateToken(token) {
|
|
20
|
+
return crypto.createHash('md5').update(token).digest('hex');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Validate token format
|
|
25
|
+
*/
|
|
26
|
+
static validateToken(token) {
|
|
27
|
+
if (!token || typeof token !== 'string' || token.trim() === '') {
|
|
28
|
+
throw new Error('Token must be a non-empty string');
|
|
29
|
+
}
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
module.exports = Auth;
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hlquery Node.js Client - CSV parsing helper
|
|
3
|
+
*
|
|
4
|
+
* Copyright (C) 2021-2026, Carlos F. Ferry <carlos.ferry@gmail.com>
|
|
5
|
+
*
|
|
6
|
+
* This file is part of hlquery, released under the BSD License version 3.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const crypto = require('crypto');
|
|
12
|
+
const { ValidationException } = require('../lib/Exceptions');
|
|
13
|
+
|
|
14
|
+
class CSVParser {
|
|
15
|
+
static parseFile(filePath, options = {}) {
|
|
16
|
+
this.validateFilePath(filePath);
|
|
17
|
+
|
|
18
|
+
const resolvedPath = path.resolve(filePath);
|
|
19
|
+
const stat = fs.statSync(resolvedPath);
|
|
20
|
+
|
|
21
|
+
if (!stat.isFile()) {
|
|
22
|
+
throw new ValidationException(`CSV path is not a file: ${resolvedPath}`);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const raw = fs.readFileSync(resolvedPath, 'utf8');
|
|
26
|
+
const rows = this.parseCSV(raw, options.delimiter || ',');
|
|
27
|
+
const header = rows.length > 0 ? rows[0] : [];
|
|
28
|
+
const dataRows = rows.length > 1 ? rows.slice(1) : [];
|
|
29
|
+
const content = this.normalizeRows(dataRows.length > 0 ? dataRows : rows, options);
|
|
30
|
+
|
|
31
|
+
return {
|
|
32
|
+
filePath: resolvedPath,
|
|
33
|
+
fileName: path.basename(resolvedPath),
|
|
34
|
+
title: this.resolveTitle(resolvedPath, options),
|
|
35
|
+
content: content,
|
|
36
|
+
rowCount: dataRows.length > 0 ? dataRows.length : rows.length,
|
|
37
|
+
columnCount: header.length || (rows[0] ? rows[0].length : 0),
|
|
38
|
+
header: header,
|
|
39
|
+
sizeBytes: stat.size
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
static buildDocumentFromParsedCSV(parsed, options = {}) {
|
|
44
|
+
const contentField = options.contentField || 'content';
|
|
45
|
+
const titleField = options.titleField || 'title';
|
|
46
|
+
const document = Object.assign({}, options.document || {});
|
|
47
|
+
|
|
48
|
+
document.id = options.id || document.id || this.generateDocumentId(parsed.fileName);
|
|
49
|
+
document[titleField] = options.title || document[titleField] || parsed.title;
|
|
50
|
+
document[contentField] = parsed.content;
|
|
51
|
+
document.file_name = parsed.fileName;
|
|
52
|
+
document.file_path = parsed.filePath;
|
|
53
|
+
document.mime_type = 'text/csv';
|
|
54
|
+
document.row_count = String(parsed.rowCount);
|
|
55
|
+
document.column_count = String(parsed.columnCount);
|
|
56
|
+
|
|
57
|
+
if (parsed.header.length > 0) {
|
|
58
|
+
document.columns = parsed.header.map((column) => this.normalizeText(column, options));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return document;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
static parseCSV(input, delimiter) {
|
|
65
|
+
const rows = [];
|
|
66
|
+
let row = [];
|
|
67
|
+
let value = '';
|
|
68
|
+
let inQuotes = false;
|
|
69
|
+
|
|
70
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
71
|
+
const char = input[i];
|
|
72
|
+
const next = input[i + 1];
|
|
73
|
+
|
|
74
|
+
if (char === '"') {
|
|
75
|
+
if (inQuotes && next === '"') {
|
|
76
|
+
value += '"';
|
|
77
|
+
i += 1;
|
|
78
|
+
} else {
|
|
79
|
+
inQuotes = !inQuotes;
|
|
80
|
+
}
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (!inQuotes && char === delimiter) {
|
|
85
|
+
row.push(value);
|
|
86
|
+
value = '';
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (!inQuotes && (char === '\n' || char === '\r')) {
|
|
91
|
+
if (char === '\r' && next === '\n') {
|
|
92
|
+
i += 1;
|
|
93
|
+
}
|
|
94
|
+
row.push(value);
|
|
95
|
+
value = '';
|
|
96
|
+
if (row.length > 1 || row[0] !== '') {
|
|
97
|
+
rows.push(row);
|
|
98
|
+
}
|
|
99
|
+
row = [];
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
value += char;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (value !== '' || row.length > 0) {
|
|
107
|
+
row.push(value);
|
|
108
|
+
rows.push(row);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return rows;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
static normalizeRows(rows, options = {}) {
|
|
115
|
+
return rows
|
|
116
|
+
.map((row) => row.map((cell) => this.normalizeText(cell, options)).filter(Boolean).join(' '))
|
|
117
|
+
.filter(Boolean)
|
|
118
|
+
.join('\n')
|
|
119
|
+
.trim();
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
static normalizeText(text, options = {}) {
|
|
123
|
+
let output = String(text || '').replace(/\r\n/g, '\n').replace(/\0/g, '');
|
|
124
|
+
output = output.replace(/,/g, ' ');
|
|
125
|
+
|
|
126
|
+
if (options.collapseWhitespace !== false) {
|
|
127
|
+
output = output.replace(/[ \t]+/g, ' ');
|
|
128
|
+
output = output.replace(/\n{3,}/g, '\n\n');
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return output.trim();
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
static resolveTitle(filePath, options = {}) {
|
|
135
|
+
if (options.title) {
|
|
136
|
+
return this.normalizeText(options.title, options);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return this.normalizeText(path.basename(filePath, path.extname(filePath)), options);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
static generateDocumentId(fileName) {
|
|
143
|
+
const stem = path.basename(fileName, path.extname(fileName))
|
|
144
|
+
.toLowerCase()
|
|
145
|
+
.replace(/[^a-z0-9._-]+/g, '_')
|
|
146
|
+
.replace(/^_+|_+$/g, '')
|
|
147
|
+
.slice(0, 40);
|
|
148
|
+
const suffix = crypto.randomBytes(4).toString('hex');
|
|
149
|
+
|
|
150
|
+
return `${stem || 'csv'}_${suffix}`;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
static validateFilePath(filePath) {
|
|
154
|
+
if (!filePath || typeof filePath !== 'string') {
|
|
155
|
+
throw new ValidationException('CSV file path must be a non-empty string');
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
module.exports = CSVParser;
|
package/utils/Config.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hlquery Node.js Client - Configuration Utilities
|
|
3
|
+
*
|
|
4
|
+
* Copyright (C) 2021-2026, Carlos F. Ferry <carlos.ferry@gmail.com>
|
|
5
|
+
*
|
|
6
|
+
* This file is part of hlquery, released under the BSD License version 3.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Configuration management utilities
|
|
11
|
+
*/
|
|
12
|
+
class Config {
|
|
13
|
+
static getDefaultBaseUrl() {
|
|
14
|
+
return process.env.HLQ_BASE_URL || process.env.HLQUERY_BASE_URL || 'http://localhost:9200';
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Merge options with defaults
|
|
19
|
+
*/
|
|
20
|
+
static mergeDefaults(options = {}) {
|
|
21
|
+
return {
|
|
22
|
+
base_url: options.base_url || Config.getDefaultBaseUrl(),
|
|
23
|
+
timeout: options.timeout || 30000,
|
|
24
|
+
token: options.token || null,
|
|
25
|
+
auth_method: options.auth_method || 'bearer',
|
|
26
|
+
...options
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Normalize URL
|
|
32
|
+
*/
|
|
33
|
+
static normalizeUrl(url) {
|
|
34
|
+
if (!url) {
|
|
35
|
+
return Config.getDefaultBaseUrl();
|
|
36
|
+
}
|
|
37
|
+
url = url.trim();
|
|
38
|
+
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
|
39
|
+
url = 'http://' + url;
|
|
40
|
+
}
|
|
41
|
+
return url.replace(/\/$/, '');
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Validate URL
|
|
46
|
+
*/
|
|
47
|
+
static isValidUrl(url) {
|
|
48
|
+
try {
|
|
49
|
+
const urlObj = new URL(url);
|
|
50
|
+
return urlObj.protocol === 'http:' || urlObj.protocol === 'https:';
|
|
51
|
+
} catch (e) {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
module.exports = Config;
|