eyeling 1.27.0 → 1.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/eyeling.browser.js +123 -0
- package/eyeling.js +123 -0
- package/lib/cli.js +123 -0
- package/package.json +1 -1
- package/test/stream_messages.test.js +61 -20
|
@@ -4623,6 +4623,7 @@ module.exports = {
|
|
|
4623
4623
|
'use strict';
|
|
4624
4624
|
|
|
4625
4625
|
const fs = require('node:fs');
|
|
4626
|
+
const os = require('node:os');
|
|
4626
4627
|
const path = require('node:path');
|
|
4627
4628
|
const { pathToFileURL, fileURLToPath } = require('node:url');
|
|
4628
4629
|
const { TextDecoder } = require('node:util');
|
|
@@ -4676,6 +4677,10 @@ function __isNetworkOrFileIri(s) {
|
|
|
4676
4677
|
return typeof s === 'string' && /^(https?:|file:\/\/)/i.test(s);
|
|
4677
4678
|
}
|
|
4678
4679
|
|
|
4680
|
+
function __isHttpSource(s) {
|
|
4681
|
+
return typeof s === 'string' && /^https?:/i.test(s);
|
|
4682
|
+
}
|
|
4683
|
+
|
|
4679
4684
|
function __sourceLabelToBaseIri(sourceLabel) {
|
|
4680
4685
|
if (!sourceLabel || sourceLabel === '<stdin>') return '';
|
|
4681
4686
|
if (__isNetworkOrFileIri(sourceLabel)) return deref.stripFragment(sourceLabel);
|
|
@@ -4694,6 +4699,112 @@ function __readInputSourceSync(sourceLabel) {
|
|
|
4694
4699
|
return fs.readFileSync(sourceLabel, { encoding: 'utf8' });
|
|
4695
4700
|
}
|
|
4696
4701
|
|
|
4702
|
+
function __httpFetchScriptBody({ prefixOnly = false } = {}) {
|
|
4703
|
+
return `
|
|
4704
|
+
const fs = require('fs');
|
|
4705
|
+
const http = require('http');
|
|
4706
|
+
const https = require('https');
|
|
4707
|
+
const zlib = require('zlib');
|
|
4708
|
+
const { URL } = require('url');
|
|
4709
|
+
const urlArg = process.argv[1];
|
|
4710
|
+
const outFile = process.argv[2] || '';
|
|
4711
|
+
const limit = Math.max(1, Number(process.argv[3] || 65536));
|
|
4712
|
+
const prefixOnly = ${prefixOnly ? 'true' : 'false'};
|
|
4713
|
+
const maxRedirects = 10;
|
|
4714
|
+
function requestUrl(u, redirects) {
|
|
4715
|
+
if (redirects > maxRedirects) { console.error('Too many redirects'); process.exit(3); }
|
|
4716
|
+
const parsed = new URL(u);
|
|
4717
|
+
const mod = parsed.protocol === 'https:' ? https : parsed.protocol === 'http:' ? http : null;
|
|
4718
|
+
if (!mod) { console.error('Unsupported protocol ' + parsed.protocol); process.exit(2); }
|
|
4719
|
+
const headers = {
|
|
4720
|
+
accept: 'text/n3, text/turtle, application/trig, application/n-triples, application/n-quads, text/plain;q=0.8, */*;q=0.01',
|
|
4721
|
+
'accept-encoding': 'identity',
|
|
4722
|
+
'user-agent': 'eyeling-rdf-message-stream'
|
|
4723
|
+
};
|
|
4724
|
+
if (prefixOnly) headers.range = 'bytes=0-' + String(limit - 1);
|
|
4725
|
+
const req = mod.request({
|
|
4726
|
+
protocol: parsed.protocol,
|
|
4727
|
+
hostname: parsed.hostname,
|
|
4728
|
+
port: parsed.port || undefined,
|
|
4729
|
+
path: parsed.pathname + parsed.search,
|
|
4730
|
+
headers,
|
|
4731
|
+
}, (res) => {
|
|
4732
|
+
const sc = res.statusCode || 0;
|
|
4733
|
+
if (sc >= 300 && sc < 400 && res.headers && res.headers.location) {
|
|
4734
|
+
const next = new URL(res.headers.location, u).toString();
|
|
4735
|
+
res.resume();
|
|
4736
|
+
return requestUrl(next, redirects + 1);
|
|
4737
|
+
}
|
|
4738
|
+
if (sc < 200 || sc >= 300) {
|
|
4739
|
+
res.resume();
|
|
4740
|
+
console.error('HTTP status ' + sc);
|
|
4741
|
+
process.exit(4);
|
|
4742
|
+
}
|
|
4743
|
+
const enc = String((res.headers && res.headers['content-encoding']) || '').toLowerCase();
|
|
4744
|
+
let body = res;
|
|
4745
|
+
if (enc.includes('gzip')) body = res.pipe(zlib.createGunzip());
|
|
4746
|
+
else if (enc.includes('deflate')) body = res.pipe(zlib.createInflate());
|
|
4747
|
+
else if (enc.includes('br')) body = res.pipe(zlib.createBrotliDecompress());
|
|
4748
|
+
if (prefixOnly) {
|
|
4749
|
+
const chunks = [];
|
|
4750
|
+
let bytes = 0;
|
|
4751
|
+
let finished = false;
|
|
4752
|
+
function finish() {
|
|
4753
|
+
if (finished) return;
|
|
4754
|
+
finished = true;
|
|
4755
|
+
const buf = Buffer.concat(chunks, bytes).subarray(0, limit);
|
|
4756
|
+
process.stdout.write(buf.toString('utf8'));
|
|
4757
|
+
process.exit(0);
|
|
4758
|
+
}
|
|
4759
|
+
body.on('data', (chunk) => {
|
|
4760
|
+
if (finished) return;
|
|
4761
|
+
chunks.push(chunk);
|
|
4762
|
+
bytes += chunk.length;
|
|
4763
|
+
if (bytes >= limit) finish();
|
|
4764
|
+
});
|
|
4765
|
+
body.on('end', finish);
|
|
4766
|
+
body.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(5); });
|
|
4767
|
+
return;
|
|
4768
|
+
}
|
|
4769
|
+
const out = fs.createWriteStream(outFile);
|
|
4770
|
+
body.pipe(out);
|
|
4771
|
+
body.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(5); });
|
|
4772
|
+
out.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(6); });
|
|
4773
|
+
out.on('finish', () => process.exit(0));
|
|
4774
|
+
});
|
|
4775
|
+
req.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(5); });
|
|
4776
|
+
req.end();
|
|
4777
|
+
}
|
|
4778
|
+
requestUrl(urlArg, 0);
|
|
4779
|
+
`;
|
|
4780
|
+
}
|
|
4781
|
+
|
|
4782
|
+
function __readHttpPrefixSync(sourceLabel, byteLimit = 64 * 1024) {
|
|
4783
|
+
const cp = require('node:child_process');
|
|
4784
|
+
const r = cp.spawnSync(process.execPath, ['-e', __httpFetchScriptBody({ prefixOnly: true }), sourceLabel, '', String(byteLimit)], {
|
|
4785
|
+
encoding: 'utf8',
|
|
4786
|
+
maxBuffer: byteLimit + 16 * 1024,
|
|
4787
|
+
});
|
|
4788
|
+
if (r.status !== 0) throw new Error(`Failed to dereference ${sourceLabel}${r.stderr ? ': ' + String(r.stderr).trim() : ''}`);
|
|
4789
|
+
return r.stdout;
|
|
4790
|
+
}
|
|
4791
|
+
|
|
4792
|
+
function __downloadHttpSourceToTempFileSync(sourceLabel) {
|
|
4793
|
+
const cp = require('node:child_process');
|
|
4794
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eyeling-rdf-message-log-'));
|
|
4795
|
+
const file = path.join(dir, 'source.txt');
|
|
4796
|
+
const r = cp.spawnSync(process.execPath, ['-e', __httpFetchScriptBody({ prefixOnly: false }), sourceLabel, file], {
|
|
4797
|
+
encoding: 'utf8',
|
|
4798
|
+
maxBuffer: 1024 * 1024,
|
|
4799
|
+
stdio: ['ignore', 'ignore', 'pipe'],
|
|
4800
|
+
});
|
|
4801
|
+
if (r.status !== 0) {
|
|
4802
|
+
try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
|
|
4803
|
+
throw new Error(`Failed to dereference ${sourceLabel}${r.stderr ? ': ' + String(r.stderr).trim() : ''}`);
|
|
4804
|
+
}
|
|
4805
|
+
return { file, cleanup: () => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch {} } };
|
|
4806
|
+
}
|
|
4807
|
+
|
|
4697
4808
|
|
|
4698
4809
|
const RDF_MESSAGE_VERSION_RE = /^\s*(?:@version|VERSION)\s+(["'])(?:1\.1|1\.2|1\.2-basic)-messages\1\s*\.?\s*(?:#.*)?$/im;
|
|
4699
4810
|
const RDF_MESSAGE_VERSION_LINE_RE = /^\s*(?:@version|VERSION)\s+(["'])(?:1\.1|1\.2|1\.2-basic)-messages\1\s*\.?\s*(?:#.*)?$/i;
|
|
@@ -4749,6 +4860,9 @@ function __sourceLooksLikeRdfMessageLogSync(sourceLabel) {
|
|
|
4749
4860
|
fs.closeSync(fd);
|
|
4750
4861
|
}
|
|
4751
4862
|
}
|
|
4863
|
+
if (__isHttpSource(sourceLabel)) {
|
|
4864
|
+
return RDF_MESSAGE_VERSION_RE.test(__readHttpPrefixSync(sourceLabel));
|
|
4865
|
+
}
|
|
4752
4866
|
return RDF_MESSAGE_VERSION_RE.test(__readInputSourceSync(sourceLabel));
|
|
4753
4867
|
}
|
|
4754
4868
|
|
|
@@ -4907,6 +5021,15 @@ function __forEachRdfMessageChunkSync(sourceLabel, onMessage) {
|
|
|
4907
5021
|
forEachRdfMessageChunkInFileSync(filePath, onMessage);
|
|
4908
5022
|
return;
|
|
4909
5023
|
}
|
|
5024
|
+
if (__isHttpSource(sourceLabel)) {
|
|
5025
|
+
const tmp = __downloadHttpSourceToTempFileSync(sourceLabel);
|
|
5026
|
+
try {
|
|
5027
|
+
forEachRdfMessageChunkInFileSync(tmp.file, onMessage);
|
|
5028
|
+
} finally {
|
|
5029
|
+
tmp.cleanup();
|
|
5030
|
+
}
|
|
5031
|
+
return;
|
|
5032
|
+
}
|
|
4910
5033
|
forEachRdfMessageChunkInText(__readInputSourceSync(sourceLabel), onMessage);
|
|
4911
5034
|
}
|
|
4912
5035
|
|
package/eyeling.js
CHANGED
|
@@ -4623,6 +4623,7 @@ module.exports = {
|
|
|
4623
4623
|
'use strict';
|
|
4624
4624
|
|
|
4625
4625
|
const fs = require('node:fs');
|
|
4626
|
+
const os = require('node:os');
|
|
4626
4627
|
const path = require('node:path');
|
|
4627
4628
|
const { pathToFileURL, fileURLToPath } = require('node:url');
|
|
4628
4629
|
const { TextDecoder } = require('node:util');
|
|
@@ -4676,6 +4677,10 @@ function __isNetworkOrFileIri(s) {
|
|
|
4676
4677
|
return typeof s === 'string' && /^(https?:|file:\/\/)/i.test(s);
|
|
4677
4678
|
}
|
|
4678
4679
|
|
|
4680
|
+
function __isHttpSource(s) {
|
|
4681
|
+
return typeof s === 'string' && /^https?:/i.test(s);
|
|
4682
|
+
}
|
|
4683
|
+
|
|
4679
4684
|
function __sourceLabelToBaseIri(sourceLabel) {
|
|
4680
4685
|
if (!sourceLabel || sourceLabel === '<stdin>') return '';
|
|
4681
4686
|
if (__isNetworkOrFileIri(sourceLabel)) return deref.stripFragment(sourceLabel);
|
|
@@ -4694,6 +4699,112 @@ function __readInputSourceSync(sourceLabel) {
|
|
|
4694
4699
|
return fs.readFileSync(sourceLabel, { encoding: 'utf8' });
|
|
4695
4700
|
}
|
|
4696
4701
|
|
|
4702
|
+
function __httpFetchScriptBody({ prefixOnly = false } = {}) {
|
|
4703
|
+
return `
|
|
4704
|
+
const fs = require('fs');
|
|
4705
|
+
const http = require('http');
|
|
4706
|
+
const https = require('https');
|
|
4707
|
+
const zlib = require('zlib');
|
|
4708
|
+
const { URL } = require('url');
|
|
4709
|
+
const urlArg = process.argv[1];
|
|
4710
|
+
const outFile = process.argv[2] || '';
|
|
4711
|
+
const limit = Math.max(1, Number(process.argv[3] || 65536));
|
|
4712
|
+
const prefixOnly = ${prefixOnly ? 'true' : 'false'};
|
|
4713
|
+
const maxRedirects = 10;
|
|
4714
|
+
function requestUrl(u, redirects) {
|
|
4715
|
+
if (redirects > maxRedirects) { console.error('Too many redirects'); process.exit(3); }
|
|
4716
|
+
const parsed = new URL(u);
|
|
4717
|
+
const mod = parsed.protocol === 'https:' ? https : parsed.protocol === 'http:' ? http : null;
|
|
4718
|
+
if (!mod) { console.error('Unsupported protocol ' + parsed.protocol); process.exit(2); }
|
|
4719
|
+
const headers = {
|
|
4720
|
+
accept: 'text/n3, text/turtle, application/trig, application/n-triples, application/n-quads, text/plain;q=0.8, */*;q=0.01',
|
|
4721
|
+
'accept-encoding': 'identity',
|
|
4722
|
+
'user-agent': 'eyeling-rdf-message-stream'
|
|
4723
|
+
};
|
|
4724
|
+
if (prefixOnly) headers.range = 'bytes=0-' + String(limit - 1);
|
|
4725
|
+
const req = mod.request({
|
|
4726
|
+
protocol: parsed.protocol,
|
|
4727
|
+
hostname: parsed.hostname,
|
|
4728
|
+
port: parsed.port || undefined,
|
|
4729
|
+
path: parsed.pathname + parsed.search,
|
|
4730
|
+
headers,
|
|
4731
|
+
}, (res) => {
|
|
4732
|
+
const sc = res.statusCode || 0;
|
|
4733
|
+
if (sc >= 300 && sc < 400 && res.headers && res.headers.location) {
|
|
4734
|
+
const next = new URL(res.headers.location, u).toString();
|
|
4735
|
+
res.resume();
|
|
4736
|
+
return requestUrl(next, redirects + 1);
|
|
4737
|
+
}
|
|
4738
|
+
if (sc < 200 || sc >= 300) {
|
|
4739
|
+
res.resume();
|
|
4740
|
+
console.error('HTTP status ' + sc);
|
|
4741
|
+
process.exit(4);
|
|
4742
|
+
}
|
|
4743
|
+
const enc = String((res.headers && res.headers['content-encoding']) || '').toLowerCase();
|
|
4744
|
+
let body = res;
|
|
4745
|
+
if (enc.includes('gzip')) body = res.pipe(zlib.createGunzip());
|
|
4746
|
+
else if (enc.includes('deflate')) body = res.pipe(zlib.createInflate());
|
|
4747
|
+
else if (enc.includes('br')) body = res.pipe(zlib.createBrotliDecompress());
|
|
4748
|
+
if (prefixOnly) {
|
|
4749
|
+
const chunks = [];
|
|
4750
|
+
let bytes = 0;
|
|
4751
|
+
let finished = false;
|
|
4752
|
+
function finish() {
|
|
4753
|
+
if (finished) return;
|
|
4754
|
+
finished = true;
|
|
4755
|
+
const buf = Buffer.concat(chunks, bytes).subarray(0, limit);
|
|
4756
|
+
process.stdout.write(buf.toString('utf8'));
|
|
4757
|
+
process.exit(0);
|
|
4758
|
+
}
|
|
4759
|
+
body.on('data', (chunk) => {
|
|
4760
|
+
if (finished) return;
|
|
4761
|
+
chunks.push(chunk);
|
|
4762
|
+
bytes += chunk.length;
|
|
4763
|
+
if (bytes >= limit) finish();
|
|
4764
|
+
});
|
|
4765
|
+
body.on('end', finish);
|
|
4766
|
+
body.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(5); });
|
|
4767
|
+
return;
|
|
4768
|
+
}
|
|
4769
|
+
const out = fs.createWriteStream(outFile);
|
|
4770
|
+
body.pipe(out);
|
|
4771
|
+
body.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(5); });
|
|
4772
|
+
out.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(6); });
|
|
4773
|
+
out.on('finish', () => process.exit(0));
|
|
4774
|
+
});
|
|
4775
|
+
req.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(5); });
|
|
4776
|
+
req.end();
|
|
4777
|
+
}
|
|
4778
|
+
requestUrl(urlArg, 0);
|
|
4779
|
+
`;
|
|
4780
|
+
}
|
|
4781
|
+
|
|
4782
|
+
function __readHttpPrefixSync(sourceLabel, byteLimit = 64 * 1024) {
|
|
4783
|
+
const cp = require('node:child_process');
|
|
4784
|
+
const r = cp.spawnSync(process.execPath, ['-e', __httpFetchScriptBody({ prefixOnly: true }), sourceLabel, '', String(byteLimit)], {
|
|
4785
|
+
encoding: 'utf8',
|
|
4786
|
+
maxBuffer: byteLimit + 16 * 1024,
|
|
4787
|
+
});
|
|
4788
|
+
if (r.status !== 0) throw new Error(`Failed to dereference ${sourceLabel}${r.stderr ? ': ' + String(r.stderr).trim() : ''}`);
|
|
4789
|
+
return r.stdout;
|
|
4790
|
+
}
|
|
4791
|
+
|
|
4792
|
+
function __downloadHttpSourceToTempFileSync(sourceLabel) {
|
|
4793
|
+
const cp = require('node:child_process');
|
|
4794
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eyeling-rdf-message-log-'));
|
|
4795
|
+
const file = path.join(dir, 'source.txt');
|
|
4796
|
+
const r = cp.spawnSync(process.execPath, ['-e', __httpFetchScriptBody({ prefixOnly: false }), sourceLabel, file], {
|
|
4797
|
+
encoding: 'utf8',
|
|
4798
|
+
maxBuffer: 1024 * 1024,
|
|
4799
|
+
stdio: ['ignore', 'ignore', 'pipe'],
|
|
4800
|
+
});
|
|
4801
|
+
if (r.status !== 0) {
|
|
4802
|
+
try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
|
|
4803
|
+
throw new Error(`Failed to dereference ${sourceLabel}${r.stderr ? ': ' + String(r.stderr).trim() : ''}`);
|
|
4804
|
+
}
|
|
4805
|
+
return { file, cleanup: () => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch {} } };
|
|
4806
|
+
}
|
|
4807
|
+
|
|
4697
4808
|
|
|
4698
4809
|
const RDF_MESSAGE_VERSION_RE = /^\s*(?:@version|VERSION)\s+(["'])(?:1\.1|1\.2|1\.2-basic)-messages\1\s*\.?\s*(?:#.*)?$/im;
|
|
4699
4810
|
const RDF_MESSAGE_VERSION_LINE_RE = /^\s*(?:@version|VERSION)\s+(["'])(?:1\.1|1\.2|1\.2-basic)-messages\1\s*\.?\s*(?:#.*)?$/i;
|
|
@@ -4749,6 +4860,9 @@ function __sourceLooksLikeRdfMessageLogSync(sourceLabel) {
|
|
|
4749
4860
|
fs.closeSync(fd);
|
|
4750
4861
|
}
|
|
4751
4862
|
}
|
|
4863
|
+
if (__isHttpSource(sourceLabel)) {
|
|
4864
|
+
return RDF_MESSAGE_VERSION_RE.test(__readHttpPrefixSync(sourceLabel));
|
|
4865
|
+
}
|
|
4752
4866
|
return RDF_MESSAGE_VERSION_RE.test(__readInputSourceSync(sourceLabel));
|
|
4753
4867
|
}
|
|
4754
4868
|
|
|
@@ -4907,6 +5021,15 @@ function __forEachRdfMessageChunkSync(sourceLabel, onMessage) {
|
|
|
4907
5021
|
forEachRdfMessageChunkInFileSync(filePath, onMessage);
|
|
4908
5022
|
return;
|
|
4909
5023
|
}
|
|
5024
|
+
if (__isHttpSource(sourceLabel)) {
|
|
5025
|
+
const tmp = __downloadHttpSourceToTempFileSync(sourceLabel);
|
|
5026
|
+
try {
|
|
5027
|
+
forEachRdfMessageChunkInFileSync(tmp.file, onMessage);
|
|
5028
|
+
} finally {
|
|
5029
|
+
tmp.cleanup();
|
|
5030
|
+
}
|
|
5031
|
+
return;
|
|
5032
|
+
}
|
|
4910
5033
|
forEachRdfMessageChunkInText(__readInputSourceSync(sourceLabel), onMessage);
|
|
4911
5034
|
}
|
|
4912
5035
|
|
package/lib/cli.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
'use strict';
|
|
9
9
|
|
|
10
10
|
const fs = require('node:fs');
|
|
11
|
+
const os = require('node:os');
|
|
11
12
|
const path = require('node:path');
|
|
12
13
|
const { pathToFileURL, fileURLToPath } = require('node:url');
|
|
13
14
|
const { TextDecoder } = require('node:util');
|
|
@@ -61,6 +62,10 @@ function __isNetworkOrFileIri(s) {
|
|
|
61
62
|
return typeof s === 'string' && /^(https?:|file:\/\/)/i.test(s);
|
|
62
63
|
}
|
|
63
64
|
|
|
65
|
+
function __isHttpSource(s) {
|
|
66
|
+
return typeof s === 'string' && /^https?:/i.test(s);
|
|
67
|
+
}
|
|
68
|
+
|
|
64
69
|
function __sourceLabelToBaseIri(sourceLabel) {
|
|
65
70
|
if (!sourceLabel || sourceLabel === '<stdin>') return '';
|
|
66
71
|
if (__isNetworkOrFileIri(sourceLabel)) return deref.stripFragment(sourceLabel);
|
|
@@ -79,6 +84,112 @@ function __readInputSourceSync(sourceLabel) {
|
|
|
79
84
|
return fs.readFileSync(sourceLabel, { encoding: 'utf8' });
|
|
80
85
|
}
|
|
81
86
|
|
|
87
|
+
function __httpFetchScriptBody({ prefixOnly = false } = {}) {
|
|
88
|
+
return `
|
|
89
|
+
const fs = require('fs');
|
|
90
|
+
const http = require('http');
|
|
91
|
+
const https = require('https');
|
|
92
|
+
const zlib = require('zlib');
|
|
93
|
+
const { URL } = require('url');
|
|
94
|
+
const urlArg = process.argv[1];
|
|
95
|
+
const outFile = process.argv[2] || '';
|
|
96
|
+
const limit = Math.max(1, Number(process.argv[3] || 65536));
|
|
97
|
+
const prefixOnly = ${prefixOnly ? 'true' : 'false'};
|
|
98
|
+
const maxRedirects = 10;
|
|
99
|
+
function requestUrl(u, redirects) {
|
|
100
|
+
if (redirects > maxRedirects) { console.error('Too many redirects'); process.exit(3); }
|
|
101
|
+
const parsed = new URL(u);
|
|
102
|
+
const mod = parsed.protocol === 'https:' ? https : parsed.protocol === 'http:' ? http : null;
|
|
103
|
+
if (!mod) { console.error('Unsupported protocol ' + parsed.protocol); process.exit(2); }
|
|
104
|
+
const headers = {
|
|
105
|
+
accept: 'text/n3, text/turtle, application/trig, application/n-triples, application/n-quads, text/plain;q=0.8, */*;q=0.01',
|
|
106
|
+
'accept-encoding': 'identity',
|
|
107
|
+
'user-agent': 'eyeling-rdf-message-stream'
|
|
108
|
+
};
|
|
109
|
+
if (prefixOnly) headers.range = 'bytes=0-' + String(limit - 1);
|
|
110
|
+
const req = mod.request({
|
|
111
|
+
protocol: parsed.protocol,
|
|
112
|
+
hostname: parsed.hostname,
|
|
113
|
+
port: parsed.port || undefined,
|
|
114
|
+
path: parsed.pathname + parsed.search,
|
|
115
|
+
headers,
|
|
116
|
+
}, (res) => {
|
|
117
|
+
const sc = res.statusCode || 0;
|
|
118
|
+
if (sc >= 300 && sc < 400 && res.headers && res.headers.location) {
|
|
119
|
+
const next = new URL(res.headers.location, u).toString();
|
|
120
|
+
res.resume();
|
|
121
|
+
return requestUrl(next, redirects + 1);
|
|
122
|
+
}
|
|
123
|
+
if (sc < 200 || sc >= 300) {
|
|
124
|
+
res.resume();
|
|
125
|
+
console.error('HTTP status ' + sc);
|
|
126
|
+
process.exit(4);
|
|
127
|
+
}
|
|
128
|
+
const enc = String((res.headers && res.headers['content-encoding']) || '').toLowerCase();
|
|
129
|
+
let body = res;
|
|
130
|
+
if (enc.includes('gzip')) body = res.pipe(zlib.createGunzip());
|
|
131
|
+
else if (enc.includes('deflate')) body = res.pipe(zlib.createInflate());
|
|
132
|
+
else if (enc.includes('br')) body = res.pipe(zlib.createBrotliDecompress());
|
|
133
|
+
if (prefixOnly) {
|
|
134
|
+
const chunks = [];
|
|
135
|
+
let bytes = 0;
|
|
136
|
+
let finished = false;
|
|
137
|
+
function finish() {
|
|
138
|
+
if (finished) return;
|
|
139
|
+
finished = true;
|
|
140
|
+
const buf = Buffer.concat(chunks, bytes).subarray(0, limit);
|
|
141
|
+
process.stdout.write(buf.toString('utf8'));
|
|
142
|
+
process.exit(0);
|
|
143
|
+
}
|
|
144
|
+
body.on('data', (chunk) => {
|
|
145
|
+
if (finished) return;
|
|
146
|
+
chunks.push(chunk);
|
|
147
|
+
bytes += chunk.length;
|
|
148
|
+
if (bytes >= limit) finish();
|
|
149
|
+
});
|
|
150
|
+
body.on('end', finish);
|
|
151
|
+
body.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(5); });
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const out = fs.createWriteStream(outFile);
|
|
155
|
+
body.pipe(out);
|
|
156
|
+
body.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(5); });
|
|
157
|
+
out.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(6); });
|
|
158
|
+
out.on('finish', () => process.exit(0));
|
|
159
|
+
});
|
|
160
|
+
req.on('error', (e) => { console.error(e && e.message ? e.message : String(e)); process.exit(5); });
|
|
161
|
+
req.end();
|
|
162
|
+
}
|
|
163
|
+
requestUrl(urlArg, 0);
|
|
164
|
+
`;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function __readHttpPrefixSync(sourceLabel, byteLimit = 64 * 1024) {
|
|
168
|
+
const cp = require('node:child_process');
|
|
169
|
+
const r = cp.spawnSync(process.execPath, ['-e', __httpFetchScriptBody({ prefixOnly: true }), sourceLabel, '', String(byteLimit)], {
|
|
170
|
+
encoding: 'utf8',
|
|
171
|
+
maxBuffer: byteLimit + 16 * 1024,
|
|
172
|
+
});
|
|
173
|
+
if (r.status !== 0) throw new Error(`Failed to dereference ${sourceLabel}${r.stderr ? ': ' + String(r.stderr).trim() : ''}`);
|
|
174
|
+
return r.stdout;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function __downloadHttpSourceToTempFileSync(sourceLabel) {
|
|
178
|
+
const cp = require('node:child_process');
|
|
179
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eyeling-rdf-message-log-'));
|
|
180
|
+
const file = path.join(dir, 'source.txt');
|
|
181
|
+
const r = cp.spawnSync(process.execPath, ['-e', __httpFetchScriptBody({ prefixOnly: false }), sourceLabel, file], {
|
|
182
|
+
encoding: 'utf8',
|
|
183
|
+
maxBuffer: 1024 * 1024,
|
|
184
|
+
stdio: ['ignore', 'ignore', 'pipe'],
|
|
185
|
+
});
|
|
186
|
+
if (r.status !== 0) {
|
|
187
|
+
try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
|
|
188
|
+
throw new Error(`Failed to dereference ${sourceLabel}${r.stderr ? ': ' + String(r.stderr).trim() : ''}`);
|
|
189
|
+
}
|
|
190
|
+
return { file, cleanup: () => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch {} } };
|
|
191
|
+
}
|
|
192
|
+
|
|
82
193
|
|
|
83
194
|
const RDF_MESSAGE_VERSION_RE = /^\s*(?:@version|VERSION)\s+(["'])(?:1\.1|1\.2|1\.2-basic)-messages\1\s*\.?\s*(?:#.*)?$/im;
|
|
84
195
|
const RDF_MESSAGE_VERSION_LINE_RE = /^\s*(?:@version|VERSION)\s+(["'])(?:1\.1|1\.2|1\.2-basic)-messages\1\s*\.?\s*(?:#.*)?$/i;
|
|
@@ -134,6 +245,9 @@ function __sourceLooksLikeRdfMessageLogSync(sourceLabel) {
|
|
|
134
245
|
fs.closeSync(fd);
|
|
135
246
|
}
|
|
136
247
|
}
|
|
248
|
+
if (__isHttpSource(sourceLabel)) {
|
|
249
|
+
return RDF_MESSAGE_VERSION_RE.test(__readHttpPrefixSync(sourceLabel));
|
|
250
|
+
}
|
|
137
251
|
return RDF_MESSAGE_VERSION_RE.test(__readInputSourceSync(sourceLabel));
|
|
138
252
|
}
|
|
139
253
|
|
|
@@ -292,6 +406,15 @@ function __forEachRdfMessageChunkSync(sourceLabel, onMessage) {
|
|
|
292
406
|
forEachRdfMessageChunkInFileSync(filePath, onMessage);
|
|
293
407
|
return;
|
|
294
408
|
}
|
|
409
|
+
if (__isHttpSource(sourceLabel)) {
|
|
410
|
+
const tmp = __downloadHttpSourceToTempFileSync(sourceLabel);
|
|
411
|
+
try {
|
|
412
|
+
forEachRdfMessageChunkInFileSync(tmp.file, onMessage);
|
|
413
|
+
} finally {
|
|
414
|
+
tmp.cleanup();
|
|
415
|
+
}
|
|
416
|
+
return;
|
|
417
|
+
}
|
|
295
418
|
forEachRdfMessageChunkInText(__readInputSourceSync(sourceLabel), onMessage);
|
|
296
419
|
}
|
|
297
420
|
|
package/package.json
CHANGED
|
@@ -23,19 +23,15 @@ function info(msg) {
|
|
|
23
23
|
function fail(msg) {
|
|
24
24
|
console.error(`${C.r}FAIL${C.n} ${msg}`);
|
|
25
25
|
}
|
|
26
|
-
|
|
27
26
|
function numberedName(index, name) {
|
|
28
27
|
return `${String(index + 1).padStart(3, '0')} ${name}`;
|
|
29
28
|
}
|
|
30
|
-
|
|
31
29
|
function msNow() {
|
|
32
30
|
return Date.now();
|
|
33
31
|
}
|
|
34
|
-
|
|
35
32
|
function mkTmpDir() {
|
|
36
33
|
return fs.mkdtempSync(path.join(os.tmpdir(), 'eyeling-stream-messages-'));
|
|
37
34
|
}
|
|
38
|
-
|
|
39
35
|
function rmrf(p) {
|
|
40
36
|
try {
|
|
41
37
|
fs.rmSync(p, { recursive: true, force: true });
|
|
@@ -49,11 +45,9 @@ function runEyeling(args, opts = {}) {
|
|
|
49
45
|
maxBuffer: opts.maxBuffer || 20 * 1024 * 1024,
|
|
50
46
|
});
|
|
51
47
|
}
|
|
52
|
-
|
|
53
48
|
function expectEyelingOk(args, opts = {}) {
|
|
54
49
|
const r = runEyeling(args, opts);
|
|
55
50
|
if (r.status === 0) return r.stdout;
|
|
56
|
-
|
|
57
51
|
throw new Error(
|
|
58
52
|
`eyeling failed with exit ${r.status}\n` +
|
|
59
53
|
`STDOUT:\n${r.stdout || ''}\n` +
|
|
@@ -77,7 +71,6 @@ function writeScopedPayloadRules(file) {
|
|
|
77
71
|
'utf8',
|
|
78
72
|
);
|
|
79
73
|
}
|
|
80
|
-
|
|
81
74
|
function writeBasicMessageLog(file) {
|
|
82
75
|
fs.writeFileSync(
|
|
83
76
|
file,
|
|
@@ -95,7 +88,6 @@ function writeBasicMessageLog(file) {
|
|
|
95
88
|
'utf8',
|
|
96
89
|
);
|
|
97
90
|
}
|
|
98
|
-
|
|
99
91
|
function writeLargeMessageLog(file, count) {
|
|
100
92
|
let text = 'VERSION "1.2-messages"\nPREFIX : <urn:test#>\n';
|
|
101
93
|
for (let i = 1; i <= count; i += 1) {
|
|
@@ -104,7 +96,6 @@ function writeLargeMessageLog(file, count) {
|
|
|
104
96
|
}
|
|
105
97
|
fs.writeFileSync(file, text, 'utf8');
|
|
106
98
|
}
|
|
107
|
-
|
|
108
99
|
function writeMarcMessageLog(file) {
|
|
109
100
|
fs.writeFileSync(
|
|
110
101
|
file,
|
|
@@ -128,6 +119,48 @@ function writeMarcMessageLog(file) {
|
|
|
128
119
|
);
|
|
129
120
|
}
|
|
130
121
|
|
|
122
|
+
function startFileServer(file) {
|
|
123
|
+
const dir = path.dirname(file);
|
|
124
|
+
const script = path.join(dir, 'server.js');
|
|
125
|
+
const portFile = path.join(dir, 'server.port');
|
|
126
|
+
fs.writeFileSync(
|
|
127
|
+
script,
|
|
128
|
+
`const fs = require('node:fs');\n` +
|
|
129
|
+
`const http = require('node:http');\n` +
|
|
130
|
+
`const file = process.argv[2];\n` +
|
|
131
|
+
`const portFile = process.argv[3];\n` +
|
|
132
|
+
`const size = fs.statSync(file).size;\n` +
|
|
133
|
+
`const server = http.createServer((req, res) => {\n` +
|
|
134
|
+
` const headers = { 'content-type': 'text/plain', 'accept-ranges': 'bytes' };\n` +
|
|
135
|
+
` if (req.method === 'HEAD') { res.writeHead(200, { ...headers, 'content-length': size }); res.end(); return; }\n` +
|
|
136
|
+
` const range = req.headers.range;\n` +
|
|
137
|
+
` if (range) {\n` +
|
|
138
|
+
` const m = /^bytes=(\\d+)-(\\d*)$/.exec(range);\n` +
|
|
139
|
+
` const start = m ? Number(m[1]) : 0;\n` +
|
|
140
|
+
` const end = m && m[2] ? Math.min(Number(m[2]), size - 1) : size - 1;\n` +
|
|
141
|
+
` res.writeHead(206, { ...headers, 'content-range': 'bytes ' + start + '-' + end + '/' + size, 'content-length': end - start + 1 });\n` +
|
|
142
|
+
` fs.createReadStream(file, { start, end }).pipe(res);\n` +
|
|
143
|
+
` return;\n` +
|
|
144
|
+
` }\n` +
|
|
145
|
+
` res.writeHead(200, { ...headers, 'content-length': size });\n` +
|
|
146
|
+
` fs.createReadStream(file).pipe(res);\n` +
|
|
147
|
+
`});\n` +
|
|
148
|
+
`server.listen(0, '127.0.0.1', () => fs.writeFileSync(portFile, String(server.address().port)));\n`,
|
|
149
|
+
'utf8',
|
|
150
|
+
);
|
|
151
|
+
const child = cp.spawn(process.execPath, [script, file, portFile], { cwd: root, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
152
|
+
const deadline = Date.now() + 5000;
|
|
153
|
+
while (Date.now() < deadline) {
|
|
154
|
+
if (fs.existsSync(portFile)) {
|
|
155
|
+
const port = fs.readFileSync(portFile, 'utf8').trim();
|
|
156
|
+
return { url: `http://127.0.0.1:${port}/messages.txt`, stop: () => child.kill() };
|
|
157
|
+
}
|
|
158
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);
|
|
159
|
+
}
|
|
160
|
+
child.kill();
|
|
161
|
+
throw new Error('test HTTP server did not start');
|
|
162
|
+
}
|
|
163
|
+
|
|
131
164
|
const cases = [
|
|
132
165
|
{
|
|
133
166
|
name: 'scoped payload rules run once per RDF Message',
|
|
@@ -136,7 +169,6 @@ const cases = [
|
|
|
136
169
|
const log = path.join(tmp, 'messages.trig');
|
|
137
170
|
writeScopedPayloadRules(rules);
|
|
138
171
|
writeBasicMessageLog(log);
|
|
139
|
-
|
|
140
172
|
const out = expectEyelingOk(['-r', '--stream-messages', rules, log]);
|
|
141
173
|
assert.equal(out, 'one\ntwo\nthree\n');
|
|
142
174
|
},
|
|
@@ -148,7 +180,6 @@ const cases = [
|
|
|
148
180
|
const log = path.join(tmp, 'messages.trig');
|
|
149
181
|
writeScopedPayloadRules(rules);
|
|
150
182
|
writeBasicMessageLog(log);
|
|
151
|
-
|
|
152
183
|
const out = expectEyelingOk(['-r', '--stream-messages', rules, log]);
|
|
153
184
|
assert.equal(out.trim().split('\n').length, 3);
|
|
154
185
|
assert.ok(!out.includes('one\none'));
|
|
@@ -161,7 +192,6 @@ const cases = [
|
|
|
161
192
|
const log = path.join(tmp, 'large.trig');
|
|
162
193
|
writeScopedPayloadRules(rules);
|
|
163
194
|
writeLargeMessageLog(log, 1000);
|
|
164
|
-
|
|
165
195
|
const out = expectEyelingOk(['-r', '--stream-messages', rules, log]);
|
|
166
196
|
const lines = out.trim().split('\n');
|
|
167
197
|
assert.equal(lines.length, 1000);
|
|
@@ -171,12 +201,30 @@ const cases = [
|
|
|
171
201
|
assert.ok(lines.includes('1000'));
|
|
172
202
|
},
|
|
173
203
|
},
|
|
204
|
+
{
|
|
205
|
+
name: 'remote text/plain RDF Message Logs are streamed via HTTP',
|
|
206
|
+
run(tmp) {
|
|
207
|
+
const rules = path.join(tmp, 'rules.n3');
|
|
208
|
+
const log = path.join(tmp, 'remote.trig');
|
|
209
|
+
writeScopedPayloadRules(rules);
|
|
210
|
+
writeLargeMessageLog(log, 25);
|
|
211
|
+
const server = startFileServer(log);
|
|
212
|
+
try {
|
|
213
|
+
const out = expectEyelingOk(['-r', '--stream-messages', server.url, rules]);
|
|
214
|
+
const lines = out.trim().split('\n');
|
|
215
|
+
assert.equal(lines.length, 25);
|
|
216
|
+
assert.equal(lines[0], '1');
|
|
217
|
+
assert.equal(lines[24], '25');
|
|
218
|
+
} finally {
|
|
219
|
+
server.stop();
|
|
220
|
+
}
|
|
221
|
+
},
|
|
222
|
+
},
|
|
174
223
|
{
|
|
175
224
|
name: 'MARC extraction rules fire over each streamed payload graph',
|
|
176
225
|
run(tmp) {
|
|
177
226
|
const log = path.join(tmp, 'marc.messages.txt');
|
|
178
227
|
writeMarcMessageLog(log);
|
|
179
|
-
|
|
180
228
|
const fixture = path.join(root, 'test', 'fixtures', 'marc-rules-stream-messages.n3');
|
|
181
229
|
const out = expectEyelingOk(['-r', '--stream-messages', fixture, log]);
|
|
182
230
|
assert.deepEqual(out.trim().split('\n').sort(), [
|
|
@@ -196,7 +244,6 @@ const cases = [
|
|
|
196
244
|
const log = path.join(tmp, 'messages.trig');
|
|
197
245
|
writeScopedPayloadRules(rules);
|
|
198
246
|
writeBasicMessageLog(log);
|
|
199
|
-
|
|
200
247
|
const r = runEyeling(['--stream-messages', rules, log]);
|
|
201
248
|
assert.notEqual(r.status, 0);
|
|
202
249
|
assert.match(r.stderr, /requires -r\/--rdf/);
|
|
@@ -207,15 +254,12 @@ const cases = [
|
|
|
207
254
|
(function main() {
|
|
208
255
|
const suiteStart = msNow();
|
|
209
256
|
info(`Running ${cases.length} stream-message tests`);
|
|
210
|
-
|
|
211
257
|
let passed = 0;
|
|
212
258
|
let failed = 0;
|
|
213
|
-
|
|
214
259
|
for (const [index, tc] of cases.entries()) {
|
|
215
260
|
const tmp = mkTmpDir();
|
|
216
261
|
const testName = numberedName(index, tc.name);
|
|
217
262
|
const start = msNow();
|
|
218
|
-
|
|
219
263
|
try {
|
|
220
264
|
tc.run(tmp);
|
|
221
265
|
ok(`${testName} ${C.dim}(${msNow() - start} ms)${C.n}`);
|
|
@@ -228,16 +272,13 @@ const cases = [
|
|
|
228
272
|
rmrf(tmp);
|
|
229
273
|
}
|
|
230
274
|
}
|
|
231
|
-
|
|
232
275
|
console.log('');
|
|
233
276
|
const suiteMs = msNow() - suiteStart;
|
|
234
277
|
console.log(`${C.y}==${C.n} Total elapsed: ${suiteMs} ms (${(suiteMs / 1000).toFixed(2)} s)`);
|
|
235
|
-
|
|
236
278
|
if (failed === 0) {
|
|
237
279
|
ok(`All stream-message tests passed (${passed}/${cases.length})`);
|
|
238
280
|
process.exit(0);
|
|
239
281
|
}
|
|
240
|
-
|
|
241
282
|
fail(`Some stream-message tests failed (${passed}/${cases.length})`);
|
|
242
283
|
process.exit(1);
|
|
243
284
|
})();
|