@marketeer-intelligence/framework-device 0.1.0-alpha.4 → 0.1.0-alpha.5
Sign up to get free protection for your applications and to get access to all the features.
- package/dist/data-source/MarketIndexTableParaser.d.ts +8 -0
- package/dist/data-source/MarketIndexTableParaser.d.ts.map +1 -0
- package/dist/data-source/MarketIndexTableParaser.js +62 -0
- package/dist/data-source/MarketIndexTableParaser.js.map +1 -0
- package/dist/data-source/SecurityTableParaser.d.ts +7 -7
- package/dist/data-source/SecurityTableParaser.js +71 -71
- package/dist/data-source/StaticWebpageScraper.d.ts +11 -11
- package/dist/data-source/StaticWebpageScraper.js +49 -49
- package/dist/data-source/Webpage.d.ts +10 -10
- package/dist/data-source/Webpage.js +21 -21
- package/dist/data-source/WebpageBody.d.ts +10 -10
- package/dist/data-source/WebpageBody.js +33 -33
- package/dist/data-source/WebpageUrl.d.ts +10 -10
- package/dist/data-source/WebpageUrl.js +38 -38
- package/dist/data-source/index.d.ts +7 -6
- package/dist/data-source/index.d.ts.map +1 -1
- package/dist/data-source/index.js +7 -6
- package/dist/data-source/index.js.map +1 -1
- package/dist/data-store/InstrumentMongoDbCollection.d.ts +11 -11
- package/dist/data-store/InstrumentMongoDbCollection.js +57 -57
- package/dist/data-store/SecurityMongoDbCollection.d.ts +11 -11
- package/dist/data-store/SecurityMongoDbCollection.js +66 -66
- package/dist/data-store/index.d.ts +3 -3
- package/dist/data-store/index.js +3 -3
- package/dist/index.d.ts +4 -4
- package/dist/index.js +4 -4
- package/dist/task/ConfigureEnvironmentVariables.d.ts +9 -9
- package/dist/task/ConfigureEnvironmentVariables.js +31 -31
- package/dist/task/ConnectToMongoDb.d.ts +8 -8
- package/dist/task/ConnectToMongoDb.js +28 -28
- package/dist/task/DisconnectFromMongoDb.d.ts +7 -7
- package/dist/task/DisconnectFromMongoDb.js +25 -25
- package/dist/task/MongoDbUri.d.ts +10 -10
- package/dist/task/MongoDbUri.js +38 -38
- package/dist/task/index.d.ts +5 -5
- package/dist/task/index.js +5 -5
- package/package.json +4 -4
@@ -0,0 +1,8 @@
|
|
1
|
+
import { IWebpageBodyParser, Feedback, WebpageBody, Instrument } from './index.js';
|
2
|
+
export declare class MarketIndexTableParaser implements IWebpageBodyParser<Instrument[]> {
|
3
|
+
private constructor();
|
4
|
+
private static _instance;
|
5
|
+
static create(): MarketIndexTableParaser;
|
6
|
+
parse(webpageBody: WebpageBody): Promise<Feedback<Instrument[], Error>>;
|
7
|
+
}
|
8
|
+
//# sourceMappingURL=MarketIndexTableParaser.d.ts.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"MarketIndexTableParaser.d.ts","sourceRoot":"","sources":["../../src/data-source/MarketIndexTableParaser.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,kBAAkB,EAClB,QAAQ,EAIR,WAAW,EACX,UAAU,EACX,MAAM,YAAY,CAAA;AAYnB,qBAAa,uBACX,YAAW,kBAAkB,CAAC,UAAU,EAAE,CAAC;IAE3C,OAAO;IACP,OAAO,CAAC,MAAM,CAAC,SAAS,CAAyB;IACjD,MAAM,CAAC,MAAM,IAAI,uBAAuB;IAGlC,KAAK,CACT,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;CA+C1C"}
|
@@ -0,0 +1,62 @@
|
|
1
|
+
import { Failure, isFailure, Result, Instrument, } from './index.js';
|
2
|
+
import { JSDOM } from 'jsdom';
|
3
|
+
import { DateTime } from 'luxon';
|
4
|
+
function isRowValue(candidate) {
|
5
|
+
if (!Array.isArray(candidate))
|
6
|
+
return false;
|
7
|
+
if (candidate.length !== 6)
|
8
|
+
return false;
|
9
|
+
return candidate.every(element => typeof element === 'string');
|
10
|
+
}
|
11
|
+
export class MarketIndexTableParaser {
|
12
|
+
constructor() { }
|
13
|
+
static _instance;
|
14
|
+
static create() {
|
15
|
+
return this._instance || (this._instance = new MarketIndexTableParaser());
|
16
|
+
}
|
17
|
+
async parse(webpageBody) {
|
18
|
+
try {
|
19
|
+
const dom = new JSDOM(webpageBody.value);
|
20
|
+
const tableOrNull = dom.window.document.querySelector('table.h4');
|
21
|
+
if (tableOrNull === null) {
|
22
|
+
const msg = 'did not find a table element for parsing';
|
23
|
+
const error = new Error(msg);
|
24
|
+
return Failure.create(error);
|
25
|
+
}
|
26
|
+
const rows = tableOrNull.querySelectorAll('tbody tr:not([align="center"])');
|
27
|
+
const indices = [];
|
28
|
+
for (const [index, row] of rows.entries()) {
|
29
|
+
const cells = Array.from(row.querySelectorAll('td'));
|
30
|
+
const contents = cells.map(cell => cell.textContent === null ? '' : cell.textContent.trim());
|
31
|
+
if (!isRowValue(contents)) {
|
32
|
+
const reason = 'not an array of 6 strings';
|
33
|
+
console.log(`row ${index + 1} is skipped (${reason})`);
|
34
|
+
continue;
|
35
|
+
}
|
36
|
+
const [symbol, name] = contents[0].split(' ');
|
37
|
+
const instrumentValue = {
|
38
|
+
isin: contents[1],
|
39
|
+
symbol,
|
40
|
+
name,
|
41
|
+
cfi: contents[3],
|
42
|
+
published: DateTime.fromFormat(contents[2], 'yyyy/MM/dd').toISO(),
|
43
|
+
};
|
44
|
+
const instrumentOrError = Instrument.create(instrumentValue);
|
45
|
+
if (isFailure(instrumentOrError)) {
|
46
|
+
const { message } = instrumentOrError.error;
|
47
|
+
console.log(`row ${index + 1} is skipped (${message})`);
|
48
|
+
continue;
|
49
|
+
}
|
50
|
+
indices.push(instrumentOrError.value);
|
51
|
+
}
|
52
|
+
return Result.create(indices);
|
53
|
+
}
|
54
|
+
catch (error) {
|
55
|
+
if (error instanceof Error) {
|
56
|
+
return Failure.create(error);
|
57
|
+
}
|
58
|
+
throw error;
|
59
|
+
}
|
60
|
+
}
|
61
|
+
}
|
62
|
+
//# sourceMappingURL=MarketIndexTableParaser.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"MarketIndexTableParaser.js","sourceRoot":"","sources":["../../src/data-source/MarketIndexTableParaser.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,OAAO,EACP,SAAS,EACT,MAAM,EAEN,UAAU,GACX,MAAM,YAAY,CAAA;AACnB,OAAO,EAAC,KAAK,EAAC,MAAM,OAAO,CAAA;AAC3B,OAAO,EAAC,QAAQ,EAAC,MAAM,OAAO,CAAA;AAI9B,SAAS,UAAU,CAAC,SAAkB;IACpC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;QAAE,OAAO,KAAK,CAAA;IAC3C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACxC,OAAO,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAA;AAChE,CAAC;AAED,MAAM,OAAO,uBAAuB;IAGlC,gBAAuB,CAAC;IAChB,MAAM,CAAC,SAAS,CAAyB;IACjD,MAAM,CAAC,MAAM;QACX,OAAO,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,uBAAuB,EAAE,CAAC,CAAA;IAC3E,CAAC;IACD,KAAK,CAAC,KAAK,CACT,WAAwB;QAExB,IAAI;YACF,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;YACxC,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAA;YACjE,IAAI,WAAW,KAAK,IAAI,EAAE;gBACxB,MAAM,GAAG,GAAG,0CAA0C,CAAA;gBACtD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAA;gBAC5B,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;aAC7B;YACD,MAAM,IAAI,GAAG,WAAW,CAAC,gBAAgB,CACvC,gCAAgC,CACjC,CAAA;YACD,MAAM,OAAO,GAAiB,EAAE,CAAA;YAChC,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE;gBACzC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAA;gBACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAChC,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CACzD,CAAA;gBACD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;oBACzB,MAAM,MAAM,GAAG,2BAA2B,CAAA;oBAC1C,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,GAAG,CAAC,gBAAgB,MAAM,GAAG,CAAC,CAAA;oBACtD,SAAQ;iBACT;gBACD,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;gBAC7C,MAAM,eAAe,GAAG;oBACtB,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;oBACjB,MAAM;oBACN,IAAI;oBACJ,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAChB,SAAS,EAAE,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,KAAK,EAAE;iBAClE,CAAA;gBACD,MAAM,iBAAiB,GAAG,UAAU,CAAC,MAAM,CAAC,eAAe,CAAC,CAAA;gBAC5D,IAAI,SAAS,CAAC,iBAAiB,CAAC,EAAE;oBAChC,MAAM,EAAC,OAAO,EAAC,GAAG,iBAAiB,CAAC,KAAK,CAAA;oBACzC,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,GAAG,CAAC,gBAAgB,OAAO,GAAG,CAAC,CAAA;oBACvD,SAAQ;iBACT;gBACD,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAA;aACtC;YACD,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;SAC9B;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,KAAK,EAAE;gBAC1B,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;aAC7B;YACD,MAAM,KAAK,CAAA;SACZ;IACH,CAAC;CACF"}
|
@@ -1,8 +1,8 @@
|
|
1
|
-
import { Security, IWebpageBodyParser, Feedback, WebpageBody } from './index.js';
|
2
|
-
export declare class SecurityTableParaser implements IWebpageBodyParser<Security[]> {
|
3
|
-
private constructor();
|
4
|
-
private static _instance;
|
5
|
-
static create(): SecurityTableParaser;
|
6
|
-
parse(webpageBody: WebpageBody): Promise<Feedback<Security[], Error>>;
|
7
|
-
}
|
1
|
+
import { Security, IWebpageBodyParser, Feedback, WebpageBody } from './index.js';
|
2
|
+
export declare class SecurityTableParaser implements IWebpageBodyParser<Security[]> {
|
3
|
+
private constructor();
|
4
|
+
private static _instance;
|
5
|
+
static create(): SecurityTableParaser;
|
6
|
+
parse(webpageBody: WebpageBody): Promise<Feedback<Security[], Error>>;
|
7
|
+
}
|
8
8
|
//# sourceMappingURL=SecurityTableParaser.d.ts.map
|
@@ -1,72 +1,72 @@
|
|
1
|
-
import { Security, Failure, isFailure, Result, } from './index.js';
|
2
|
-
import { JSDOM } from 'jsdom';
|
3
|
-
import { DateTime } from 'luxon';
|
4
|
-
function isRowValue(candidate) {
|
5
|
-
if (!Array.isArray(candidate))
|
6
|
-
return false;
|
7
|
-
if (candidate.length !== 10)
|
8
|
-
return false;
|
9
|
-
return candidate.every(element => typeof element === 'string');
|
10
|
-
}
|
11
|
-
export class SecurityTableParaser {
|
12
|
-
constructor() { }
|
13
|
-
static _instance;
|
14
|
-
static create() {
|
15
|
-
return this._instance || (this._instance = new SecurityTableParaser());
|
16
|
-
}
|
17
|
-
async parse(webpageBody) {
|
18
|
-
try {
|
19
|
-
const dom = new JSDOM(webpageBody.value);
|
20
|
-
const tableOrNull = dom.window.document.querySelector('table.h4');
|
21
|
-
if (tableOrNull === null) {
|
22
|
-
const msg = 'did not find a table element for parsing';
|
23
|
-
const error = new Error(msg);
|
24
|
-
return Failure.create(error);
|
25
|
-
}
|
26
|
-
const rows = tableOrNull.querySelectorAll('tbody tr:not([align="center"])');
|
27
|
-
const securities = [];
|
28
|
-
for (const [index, row] of rows.entries()) {
|
29
|
-
const cells = Array.from(row.querySelectorAll('td'));
|
30
|
-
const contents = cells.map(cell => cell.textContent === null ? '' : cell.textContent.trim());
|
31
|
-
if (!isRowValue(contents)) {
|
32
|
-
const reason = 'not an array of 10 strings';
|
33
|
-
console.log(`row ${index + 1} is skipped (${reason})`);
|
34
|
-
continue;
|
35
|
-
}
|
36
|
-
const securityValue = {
|
37
|
-
instrument: {
|
38
|
-
isin: contents[1],
|
39
|
-
symbol: contents[2],
|
40
|
-
name: contents[3],
|
41
|
-
cfi: contents[8],
|
42
|
-
published: DateTime.fromFormat(contents[7], 'yyyy/MM/dd').toISO(),
|
43
|
-
},
|
44
|
-
};
|
45
|
-
const withTypeValue = contents[5] === ''
|
46
|
-
? { ...securityValue }
|
47
|
-
: { ...securityValue, type: contents[5] };
|
48
|
-
const withMarketValue = contents[4] === ''
|
49
|
-
? { ...withTypeValue }
|
50
|
-
: { ...withTypeValue, market: contents[4] };
|
51
|
-
const withIndustryValue = contents[6] === ''
|
52
|
-
? { ...withMarketValue }
|
53
|
-
: { ...withMarketValue, industry: contents[6] };
|
54
|
-
const securityOrError = Security.create(withIndustryValue);
|
55
|
-
if (isFailure(securityOrError)) {
|
56
|
-
const { message } = securityOrError.error;
|
57
|
-
console.log(`row ${index + 1} is skipped (${message})`);
|
58
|
-
continue;
|
59
|
-
}
|
60
|
-
securities.push(securityOrError.value);
|
61
|
-
}
|
62
|
-
return Result.create(securities);
|
63
|
-
}
|
64
|
-
catch (error) {
|
65
|
-
if (error instanceof Error) {
|
66
|
-
return Failure.create(error);
|
67
|
-
}
|
68
|
-
throw error;
|
69
|
-
}
|
70
|
-
}
|
71
|
-
}
|
1
|
+
import { Security, Failure, isFailure, Result, } from './index.js';
|
2
|
+
import { JSDOM } from 'jsdom';
|
3
|
+
import { DateTime } from 'luxon';
|
4
|
+
function isRowValue(candidate) {
|
5
|
+
if (!Array.isArray(candidate))
|
6
|
+
return false;
|
7
|
+
if (candidate.length !== 10)
|
8
|
+
return false;
|
9
|
+
return candidate.every(element => typeof element === 'string');
|
10
|
+
}
|
11
|
+
export class SecurityTableParaser {
|
12
|
+
constructor() { }
|
13
|
+
static _instance;
|
14
|
+
static create() {
|
15
|
+
return this._instance || (this._instance = new SecurityTableParaser());
|
16
|
+
}
|
17
|
+
async parse(webpageBody) {
|
18
|
+
try {
|
19
|
+
const dom = new JSDOM(webpageBody.value);
|
20
|
+
const tableOrNull = dom.window.document.querySelector('table.h4');
|
21
|
+
if (tableOrNull === null) {
|
22
|
+
const msg = 'did not find a table element for parsing';
|
23
|
+
const error = new Error(msg);
|
24
|
+
return Failure.create(error);
|
25
|
+
}
|
26
|
+
const rows = tableOrNull.querySelectorAll('tbody tr:not([align="center"])');
|
27
|
+
const securities = [];
|
28
|
+
for (const [index, row] of rows.entries()) {
|
29
|
+
const cells = Array.from(row.querySelectorAll('td'));
|
30
|
+
const contents = cells.map(cell => cell.textContent === null ? '' : cell.textContent.trim());
|
31
|
+
if (!isRowValue(contents)) {
|
32
|
+
const reason = 'not an array of 10 strings';
|
33
|
+
console.log(`row ${index + 1} is skipped (${reason})`);
|
34
|
+
continue;
|
35
|
+
}
|
36
|
+
const securityValue = {
|
37
|
+
instrument: {
|
38
|
+
isin: contents[1],
|
39
|
+
symbol: contents[2],
|
40
|
+
name: contents[3],
|
41
|
+
cfi: contents[8],
|
42
|
+
published: DateTime.fromFormat(contents[7], 'yyyy/MM/dd').toISO(),
|
43
|
+
},
|
44
|
+
};
|
45
|
+
const withTypeValue = contents[5] === ''
|
46
|
+
? { ...securityValue }
|
47
|
+
: { ...securityValue, type: contents[5] };
|
48
|
+
const withMarketValue = contents[4] === ''
|
49
|
+
? { ...withTypeValue }
|
50
|
+
: { ...withTypeValue, market: contents[4] };
|
51
|
+
const withIndustryValue = contents[6] === ''
|
52
|
+
? { ...withMarketValue }
|
53
|
+
: { ...withMarketValue, industry: contents[6] };
|
54
|
+
const securityOrError = Security.create(withIndustryValue);
|
55
|
+
if (isFailure(securityOrError)) {
|
56
|
+
const { message } = securityOrError.error;
|
57
|
+
console.log(`row ${index + 1} is skipped (${message})`);
|
58
|
+
continue;
|
59
|
+
}
|
60
|
+
securities.push(securityOrError.value);
|
61
|
+
}
|
62
|
+
return Result.create(securities);
|
63
|
+
}
|
64
|
+
catch (error) {
|
65
|
+
if (error instanceof Error) {
|
66
|
+
return Failure.create(error);
|
67
|
+
}
|
68
|
+
throw error;
|
69
|
+
}
|
70
|
+
}
|
71
|
+
}
|
72
72
|
//# sourceMappingURL=SecurityTableParaser.js.map
|
@@ -1,12 +1,12 @@
|
|
1
|
-
import { Feedback, IValueObject, IWebpageScraper, WebpageUrlValue, WebpageBody } from './index.js';
|
2
|
-
export interface IWebpageBodyParser<DATA> {
|
3
|
-
parse(webpageBody: WebpageBody): Promise<Feedback<DATA, Error>>;
|
4
|
-
}
|
5
|
-
export declare class StaticWebpageScraper<DATA> implements IWebpageScraper<DATA> {
|
6
|
-
private _url;
|
7
|
-
private bodyParser;
|
8
|
-
private constructor();
|
9
|
-
static create<DATA>(url: IValueObject<WebpageUrlValue>, bodyParser: IWebpageBodyParser<DATA>): StaticWebpageScraper<DATA>;
|
10
|
-
scrapPage(): Promise<Feedback<DATA, Error>>;
|
11
|
-
}
|
1
|
+
import { Feedback, IValueObject, IWebpageScraper, WebpageUrlValue, WebpageBody } from './index.js';
|
2
|
+
export interface IWebpageBodyParser<DATA> {
|
3
|
+
parse(webpageBody: WebpageBody): Promise<Feedback<DATA, Error>>;
|
4
|
+
}
|
5
|
+
export declare class StaticWebpageScraper<DATA> implements IWebpageScraper<DATA> {
|
6
|
+
private _url;
|
7
|
+
private bodyParser;
|
8
|
+
private constructor();
|
9
|
+
static create<DATA>(url: IValueObject<WebpageUrlValue>, bodyParser: IWebpageBodyParser<DATA>): StaticWebpageScraper<DATA>;
|
10
|
+
scrapPage(): Promise<Feedback<DATA, Error>>;
|
11
|
+
}
|
12
12
|
//# sourceMappingURL=StaticWebpageScraper.d.ts.map
|
@@ -1,50 +1,50 @@
|
|
1
|
-
import { Failure, WebpageBody, isFailure, } from './index.js';
|
2
|
-
import got from 'got';
|
3
|
-
import iconv from 'iconv-lite';
|
4
|
-
export class StaticWebpageScraper {
|
5
|
-
_url;
|
6
|
-
bodyParser;
|
7
|
-
constructor(_url, bodyParser) {
|
8
|
-
this._url = _url;
|
9
|
-
this.bodyParser = bodyParser;
|
10
|
-
}
|
11
|
-
static create(url, bodyParser) {
|
12
|
-
return new StaticWebpageScraper(url, bodyParser);
|
13
|
-
}
|
14
|
-
async scrapPage() {
|
15
|
-
try {
|
16
|
-
const response = await got(this._url.value, { responseType: 'buffer' });
|
17
|
-
if (response.statusCode !== 200) {
|
18
|
-
const { statusCode } = response;
|
19
|
-
const statusMessage = response.statusMessage || 'status message is unavailable';
|
20
|
-
const msg = `${statusCode}: ${statusMessage}`;
|
21
|
-
const error = new Error(msg);
|
22
|
-
return Failure.create(error);
|
23
|
-
}
|
24
|
-
const contentType = typeof response.headers['content-type'] === 'string'
|
25
|
-
? response.headers['content-type'].toLowerCase()
|
26
|
-
: undefined;
|
27
|
-
const encoding = contentType === undefined || !contentType.includes('charset=')
|
28
|
-
? 'utf-8'
|
29
|
-
: contentType.split('charset=')[1].split(';')[0];
|
30
|
-
if (!iconv.encodingExists(encoding)) {
|
31
|
-
const msg = `${this._url.value} is encoded with unsupported "${encoding}" encoding`;
|
32
|
-
const error = new Error(msg);
|
33
|
-
return Failure.create(error);
|
34
|
-
}
|
35
|
-
const decoded = iconv.decode(response.body, encoding);
|
36
|
-
const webpageBodyOrError = WebpageBody.create(decoded);
|
37
|
-
if (isFailure(webpageBodyOrError))
|
38
|
-
return webpageBodyOrError;
|
39
|
-
const { value: webpageBody } = webpageBodyOrError;
|
40
|
-
return await this.bodyParser.parse(webpageBody);
|
41
|
-
}
|
42
|
-
catch (error) {
|
43
|
-
if (error instanceof Error) {
|
44
|
-
return Failure.create(error);
|
45
|
-
}
|
46
|
-
throw error;
|
47
|
-
}
|
48
|
-
}
|
49
|
-
}
|
1
|
+
import { Failure, WebpageBody, isFailure, } from './index.js';
|
2
|
+
import got from 'got';
|
3
|
+
import iconv from 'iconv-lite';
|
4
|
+
export class StaticWebpageScraper {
|
5
|
+
_url;
|
6
|
+
bodyParser;
|
7
|
+
constructor(_url, bodyParser) {
|
8
|
+
this._url = _url;
|
9
|
+
this.bodyParser = bodyParser;
|
10
|
+
}
|
11
|
+
static create(url, bodyParser) {
|
12
|
+
return new StaticWebpageScraper(url, bodyParser);
|
13
|
+
}
|
14
|
+
async scrapPage() {
|
15
|
+
try {
|
16
|
+
const response = await got(this._url.value, { responseType: 'buffer' });
|
17
|
+
if (response.statusCode !== 200) {
|
18
|
+
const { statusCode } = response;
|
19
|
+
const statusMessage = response.statusMessage || 'status message is unavailable';
|
20
|
+
const msg = `${statusCode}: ${statusMessage}`;
|
21
|
+
const error = new Error(msg);
|
22
|
+
return Failure.create(error);
|
23
|
+
}
|
24
|
+
const contentType = typeof response.headers['content-type'] === 'string'
|
25
|
+
? response.headers['content-type'].toLowerCase()
|
26
|
+
: undefined;
|
27
|
+
const encoding = contentType === undefined || !contentType.includes('charset=')
|
28
|
+
? 'utf-8'
|
29
|
+
: contentType.split('charset=')[1].split(';')[0];
|
30
|
+
if (!iconv.encodingExists(encoding)) {
|
31
|
+
const msg = `${this._url.value} is encoded with unsupported "${encoding}" encoding`;
|
32
|
+
const error = new Error(msg);
|
33
|
+
return Failure.create(error);
|
34
|
+
}
|
35
|
+
const decoded = iconv.decode(response.body, encoding);
|
36
|
+
const webpageBodyOrError = WebpageBody.create(decoded);
|
37
|
+
if (isFailure(webpageBodyOrError))
|
38
|
+
return webpageBodyOrError;
|
39
|
+
const { value: webpageBody } = webpageBodyOrError;
|
40
|
+
return await this.bodyParser.parse(webpageBody);
|
41
|
+
}
|
42
|
+
catch (error) {
|
43
|
+
if (error instanceof Error) {
|
44
|
+
return Failure.create(error);
|
45
|
+
}
|
46
|
+
throw error;
|
47
|
+
}
|
48
|
+
}
|
49
|
+
}
|
50
50
|
//# sourceMappingURL=StaticWebpageScraper.js.map
|
@@ -1,11 +1,11 @@
|
|
1
|
-
import { Feedback, IFetchable } from './index.js';
|
2
|
-
export interface IWebpageScraper<DATA> {
|
3
|
-
scrapPage(): Promise<Feedback<DATA, Error>>;
|
4
|
-
}
|
5
|
-
export declare class Webpage<DATA> implements IFetchable<DATA> {
|
6
|
-
private scraper;
|
7
|
-
private constructor();
|
8
|
-
static create<DATA>(scraper: IWebpageScraper<DATA>): Webpage<DATA>;
|
9
|
-
fetchData(): Promise<Feedback<DATA, Error>>;
|
10
|
-
}
|
1
|
+
import { Feedback, IFetchable } from './index.js';
|
2
|
+
export interface IWebpageScraper<DATA> {
|
3
|
+
scrapPage(): Promise<Feedback<DATA, Error>>;
|
4
|
+
}
|
5
|
+
export declare class Webpage<DATA> implements IFetchable<DATA> {
|
6
|
+
private scraper;
|
7
|
+
private constructor();
|
8
|
+
static create<DATA>(scraper: IWebpageScraper<DATA>): Webpage<DATA>;
|
9
|
+
fetchData(): Promise<Feedback<DATA, Error>>;
|
10
|
+
}
|
11
11
|
//# sourceMappingURL=Webpage.d.ts.map
|
@@ -1,22 +1,22 @@
|
|
1
|
-
import { Failure } from './index.js';
|
2
|
-
export class Webpage {
|
3
|
-
scraper;
|
4
|
-
constructor(scraper) {
|
5
|
-
this.scraper = scraper;
|
6
|
-
}
|
7
|
-
static create(scraper) {
|
8
|
-
return new Webpage(scraper);
|
9
|
-
}
|
10
|
-
async fetchData() {
|
11
|
-
try {
|
12
|
-
return await this.scraper.scrapPage();
|
13
|
-
}
|
14
|
-
catch (error) {
|
15
|
-
if (error instanceof Error) {
|
16
|
-
return Failure.create(error);
|
17
|
-
}
|
18
|
-
throw error;
|
19
|
-
}
|
20
|
-
}
|
21
|
-
}
|
1
|
+
import { Failure } from './index.js';
|
2
|
+
export class Webpage {
|
3
|
+
scraper;
|
4
|
+
constructor(scraper) {
|
5
|
+
this.scraper = scraper;
|
6
|
+
}
|
7
|
+
static create(scraper) {
|
8
|
+
return new Webpage(scraper);
|
9
|
+
}
|
10
|
+
async fetchData() {
|
11
|
+
try {
|
12
|
+
return await this.scraper.scrapPage();
|
13
|
+
}
|
14
|
+
catch (error) {
|
15
|
+
if (error instanceof Error) {
|
16
|
+
return Failure.create(error);
|
17
|
+
}
|
18
|
+
throw error;
|
19
|
+
}
|
20
|
+
}
|
21
|
+
}
|
22
22
|
//# sourceMappingURL=Webpage.js.map
|
@@ -1,11 +1,11 @@
|
|
1
|
-
import { Feedback, IValueObject } from './index.js';
|
2
|
-
export type WebpageBodyValue = Readonly<string>;
|
3
|
-
export declare function isWebpageBodyValue(candidate: unknown): candidate is WebpageBodyValue;
|
4
|
-
export declare class WebpageBody implements IValueObject<WebpageBodyValue> {
|
5
|
-
private _value;
|
6
|
-
private constructor();
|
7
|
-
static create(candidate: unknown): Feedback<WebpageBody, Error>;
|
8
|
-
get value(): WebpageBodyValue;
|
9
|
-
}
|
10
|
-
export declare function isWebpageBody(candidate: unknown): candidate is WebpageBody;
|
1
|
+
import { Feedback, IValueObject } from './index.js';
|
2
|
+
export type WebpageBodyValue = Readonly<string>;
|
3
|
+
export declare function isWebpageBodyValue(candidate: unknown): candidate is WebpageBodyValue;
|
4
|
+
export declare class WebpageBody implements IValueObject<WebpageBodyValue> {
|
5
|
+
private _value;
|
6
|
+
private constructor();
|
7
|
+
static create(candidate: unknown): Feedback<WebpageBody, Error>;
|
8
|
+
get value(): WebpageBodyValue;
|
9
|
+
}
|
10
|
+
export declare function isWebpageBody(candidate: unknown): candidate is WebpageBody;
|
11
11
|
//# sourceMappingURL=WebpageBody.d.ts.map
|
@@ -1,34 +1,34 @@
|
|
1
|
-
import { Failure, Result } from './index.js';
|
2
|
-
export function isWebpageBodyValue(candidate) {
|
3
|
-
return typeof candidate === 'string';
|
4
|
-
}
|
5
|
-
export class WebpageBody {
|
6
|
-
_value;
|
7
|
-
constructor(_value) {
|
8
|
-
this._value = _value;
|
9
|
-
}
|
10
|
-
static create(candidate) {
|
11
|
-
try {
|
12
|
-
if (!isWebpageBodyValue(candidate)) {
|
13
|
-
const msg = 'invalid WebpageBody creation value';
|
14
|
-
const error = new Error(msg);
|
15
|
-
return Failure.create(error);
|
16
|
-
}
|
17
|
-
const webpageBody = new WebpageBody(candidate);
|
18
|
-
return Result.create(webpageBody);
|
19
|
-
}
|
20
|
-
catch (error) {
|
21
|
-
if (error instanceof Error) {
|
22
|
-
return Failure.create(error);
|
23
|
-
}
|
24
|
-
throw error;
|
25
|
-
}
|
26
|
-
}
|
27
|
-
get value() {
|
28
|
-
return this._value;
|
29
|
-
}
|
30
|
-
}
|
31
|
-
export function isWebpageBody(candidate) {
|
32
|
-
return candidate instanceof WebpageBody;
|
33
|
-
}
|
1
|
+
import { Failure, Result } from './index.js';
|
2
|
+
export function isWebpageBodyValue(candidate) {
|
3
|
+
return typeof candidate === 'string';
|
4
|
+
}
|
5
|
+
export class WebpageBody {
|
6
|
+
_value;
|
7
|
+
constructor(_value) {
|
8
|
+
this._value = _value;
|
9
|
+
}
|
10
|
+
static create(candidate) {
|
11
|
+
try {
|
12
|
+
if (!isWebpageBodyValue(candidate)) {
|
13
|
+
const msg = 'invalid WebpageBody creation value';
|
14
|
+
const error = new Error(msg);
|
15
|
+
return Failure.create(error);
|
16
|
+
}
|
17
|
+
const webpageBody = new WebpageBody(candidate);
|
18
|
+
return Result.create(webpageBody);
|
19
|
+
}
|
20
|
+
catch (error) {
|
21
|
+
if (error instanceof Error) {
|
22
|
+
return Failure.create(error);
|
23
|
+
}
|
24
|
+
throw error;
|
25
|
+
}
|
26
|
+
}
|
27
|
+
get value() {
|
28
|
+
return this._value;
|
29
|
+
}
|
30
|
+
}
|
31
|
+
export function isWebpageBody(candidate) {
|
32
|
+
return candidate instanceof WebpageBody;
|
33
|
+
}
|
34
34
|
//# sourceMappingURL=WebpageBody.js.map
|
@@ -1,11 +1,11 @@
|
|
1
|
-
import { Feedback, IValueObject } from './index.js';
|
2
|
-
export type WebpageUrlValue = Readonly<string>;
|
3
|
-
export declare function isWebpageUrlValue(candidate: unknown): candidate is WebpageUrlValue;
|
4
|
-
export declare class WebpageUrl implements IValueObject<WebpageUrlValue> {
|
5
|
-
private _value;
|
6
|
-
private constructor();
|
7
|
-
static create(candidate: unknown): Feedback<WebpageUrl, Error>;
|
8
|
-
get value(): WebpageUrlValue;
|
9
|
-
}
|
10
|
-
export declare function isWebpageUrl(candidate: unknown): candidate is WebpageUrl;
|
1
|
+
import { Feedback, IValueObject } from './index.js';
|
2
|
+
export type WebpageUrlValue = Readonly<string>;
|
3
|
+
export declare function isWebpageUrlValue(candidate: unknown): candidate is WebpageUrlValue;
|
4
|
+
export declare class WebpageUrl implements IValueObject<WebpageUrlValue> {
|
5
|
+
private _value;
|
6
|
+
private constructor();
|
7
|
+
static create(candidate: unknown): Feedback<WebpageUrl, Error>;
|
8
|
+
get value(): WebpageUrlValue;
|
9
|
+
}
|
10
|
+
export declare function isWebpageUrl(candidate: unknown): candidate is WebpageUrl;
|
11
11
|
//# sourceMappingURL=WebpageUrl.d.ts.map
|
@@ -1,39 +1,39 @@
|
|
1
|
-
import { Failure, Result } from './index.js';
|
2
|
-
const testPattern = /^(https?):\/\/([\w.-]+\.[a-z]{2,})(:[0-9]+)?([\/\w.-]*)?(\?[\/\w=&%-]*)?(#[\/\w]*)?$/;
|
3
|
-
export function isWebpageUrlValue(candidate) {
|
4
|
-
if (typeof candidate !== 'string')
|
5
|
-
return false;
|
6
|
-
if (!testPattern.test(candidate))
|
7
|
-
return false;
|
8
|
-
return true;
|
9
|
-
}
|
10
|
-
export class WebpageUrl {
|
11
|
-
_value;
|
12
|
-
constructor(_value) {
|
13
|
-
this._value = _value;
|
14
|
-
}
|
15
|
-
static create(candidate) {
|
16
|
-
try {
|
17
|
-
if (!isWebpageUrlValue(candidate)) {
|
18
|
-
const msg = 'invalid WebpageUrl creation value';
|
19
|
-
const error = new Error(msg);
|
20
|
-
return Failure.create(error);
|
21
|
-
}
|
22
|
-
const webpageUrl = new WebpageUrl(candidate);
|
23
|
-
return Result.create(webpageUrl);
|
24
|
-
}
|
25
|
-
catch (error) {
|
26
|
-
if (error instanceof Error) {
|
27
|
-
return Failure.create(error);
|
28
|
-
}
|
29
|
-
throw error;
|
30
|
-
}
|
31
|
-
}
|
32
|
-
get value() {
|
33
|
-
return this._value;
|
34
|
-
}
|
35
|
-
}
|
36
|
-
export function isWebpageUrl(candidate) {
|
37
|
-
return candidate instanceof WebpageUrl;
|
38
|
-
}
|
1
|
+
import { Failure, Result } from './index.js';
|
2
|
+
const testPattern = /^(https?):\/\/([\w.-]+\.[a-z]{2,})(:[0-9]+)?([\/\w.-]*)?(\?[\/\w=&%-]*)?(#[\/\w]*)?$/;
|
3
|
+
export function isWebpageUrlValue(candidate) {
|
4
|
+
if (typeof candidate !== 'string')
|
5
|
+
return false;
|
6
|
+
if (!testPattern.test(candidate))
|
7
|
+
return false;
|
8
|
+
return true;
|
9
|
+
}
|
10
|
+
export class WebpageUrl {
|
11
|
+
_value;
|
12
|
+
constructor(_value) {
|
13
|
+
this._value = _value;
|
14
|
+
}
|
15
|
+
static create(candidate) {
|
16
|
+
try {
|
17
|
+
if (!isWebpageUrlValue(candidate)) {
|
18
|
+
const msg = 'invalid WebpageUrl creation value';
|
19
|
+
const error = new Error(msg);
|
20
|
+
return Failure.create(error);
|
21
|
+
}
|
22
|
+
const webpageUrl = new WebpageUrl(candidate);
|
23
|
+
return Result.create(webpageUrl);
|
24
|
+
}
|
25
|
+
catch (error) {
|
26
|
+
if (error instanceof Error) {
|
27
|
+
return Failure.create(error);
|
28
|
+
}
|
29
|
+
throw error;
|
30
|
+
}
|
31
|
+
}
|
32
|
+
get value() {
|
33
|
+
return this._value;
|
34
|
+
}
|
35
|
+
}
|
36
|
+
export function isWebpageUrl(candidate) {
|
37
|
+
return candidate instanceof WebpageUrl;
|
38
|
+
}
|
39
39
|
//# sourceMappingURL=WebpageUrl.js.map
|
@@ -1,7 +1,8 @@
|
|
1
|
-
export * from '../index.js';
|
2
|
-
export {
|
3
|
-
export {
|
4
|
-
export {
|
5
|
-
export {
|
6
|
-
export {
|
1
|
+
export * from '../index.js';
|
2
|
+
export { MarketIndexTableParaser } from './MarketIndexTableParaser.js';
|
3
|
+
export { SecurityTableParaser } from './SecurityTableParaser.js';
|
4
|
+
export { IWebpageBodyParser, StaticWebpageScraper, } from './StaticWebpageScraper.js';
|
5
|
+
export { IWebpageScraper, Webpage } from './Webpage.js';
|
6
|
+
export { WebpageBodyValue, isWebpageBodyValue, WebpageBody, isWebpageBody, } from './WebpageBody.js';
|
7
|
+
export { WebpageUrlValue, isWebpageUrlValue, WebpageUrl, isWebpageUrl, } from './WebpageUrl.js';
|
7
8
|
//# sourceMappingURL=index.d.ts.map
|