pal-crawl 1.1.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js.map +1 -1
- package/dist/pal.d.ts +17 -0
- package/dist/pal.js +50 -10
- package/dist/pal.js.map +1 -1
- package/dist/pal.test.js +62 -2
- package/dist/pal.test.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -17,9 +17,11 @@
|
|
|
17
17
|
|
|
18
18
|
- [Introduction](#introduction)
|
|
19
19
|
- [Installation](#installation)
|
|
20
|
+
- [Configuration](#configuration)
|
|
20
21
|
- [Base Types](#base-types)
|
|
21
22
|
- [Methods](#methods)
|
|
22
23
|
- [get](#get)
|
|
24
|
+
- [Examples](#examples)
|
|
23
25
|
- [License](#license)
|
|
24
26
|
|
|
25
27
|
---
|
|
@@ -38,6 +40,39 @@ npm install pal-crawl
|
|
|
38
40
|
|
|
39
41
|
---
|
|
40
42
|
|
|
43
|
+
## Configuration
|
|
44
|
+
|
|
45
|
+
`PalCrawl` 클래스는 생성자에서 선택적 설정 객체를 받을 수 있습니다.
|
|
46
|
+
|
|
47
|
+
```typescript
|
|
48
|
+
interface PalCrawlConfig {
|
|
49
|
+
userAgent?: string; // 사용자 정의 User-Agent (기본값: Chrome User-Agent)
|
|
50
|
+
timeout?: number; // HTTP 요청 타임아웃 (밀리초, 기본값: 10000)
|
|
51
|
+
retryCount?: number; // 재시도 횟수 (기본값: 3)
|
|
52
|
+
customHeaders?: Record<string, string>; // 사용자 정의 헤더
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### 기본 사용법
|
|
57
|
+
|
|
58
|
+
```typescript
|
|
59
|
+
// 기본 설정으로 사용
|
|
60
|
+
const palCrawl = new PalCrawl();
|
|
61
|
+
|
|
62
|
+
// 사용자 정의 설정으로 사용
|
|
63
|
+
const palCrawl = new PalCrawl({
|
|
64
|
+
userAgent: 'My Custom Bot 1.0',
|
|
65
|
+
timeout: 15000,
|
|
66
|
+
retryCount: 5,
|
|
67
|
+
customHeaders: {
|
|
68
|
+
'Accept-Language': 'ko-KR',
|
|
69
|
+
Accept: 'text/html,application/xhtml+xml',
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
41
76
|
## Base Types
|
|
42
77
|
|
|
43
78
|
`ITableData`는 크롤링된 입법 예고 데이터를 나타내는 인터페이스입니다.
|
|
@@ -50,6 +85,18 @@ interface ITableData {
|
|
|
50
85
|
committee: string; // 소관 위원회
|
|
51
86
|
numComments: number; // 의견 수
|
|
52
87
|
link: string; // 전문 보기 링크
|
|
88
|
+
attachments: IAttachment; // 법률안 전문 첨부파일 URL 객체
|
|
89
|
+
}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
`IAttachment`는 입법 예고의 법률안 전문 첨부파일 URL을 나타내는 인터페이스입니다.
|
|
93
|
+
|
|
94
|
+
pdf와 hwp 파일 다운로드 링크 추출을 지원합니다.
|
|
95
|
+
|
|
96
|
+
```typescript
|
|
97
|
+
interface IAttachment {
|
|
98
|
+
pdfFile: string;
|
|
99
|
+
hwpFile: string;
|
|
53
100
|
}
|
|
54
101
|
```
|
|
55
102
|
|
|
@@ -57,11 +104,11 @@ interface ITableData {
|
|
|
57
104
|
|
|
58
105
|
## Methods
|
|
59
106
|
|
|
60
|
-
### get
|
|
107
|
+
### get() => Promise<ITableData[]>
|
|
61
108
|
|
|
62
109
|
`get` 메서드는 진행 중인 입법 예고 데이터를 가져옵니다.
|
|
63
110
|
|
|
64
|
-
```
|
|
111
|
+
```typescript
|
|
65
112
|
import { PalCrawl } from 'pal-crawl';
|
|
66
113
|
|
|
67
114
|
const palCrawl = new PalCrawl();
|
|
@@ -72,6 +119,74 @@ console.log(table);
|
|
|
72
119
|
|
|
73
120
|
---
|
|
74
121
|
|
|
122
|
+
## Examples
|
|
123
|
+
|
|
124
|
+
### 기본 사용법
|
|
125
|
+
|
|
126
|
+
```typescript
|
|
127
|
+
import { PalCrawl } from 'pal-crawl';
|
|
128
|
+
|
|
129
|
+
const palCrawl = new PalCrawl();
|
|
130
|
+
const data = await palCrawl.get();
|
|
131
|
+
console.log(data);
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### 사용자 정의 User-Agent 사용
|
|
135
|
+
|
|
136
|
+
```typescript
|
|
137
|
+
import { PalCrawl, type PalCrawlConfig } from 'pal-crawl';
|
|
138
|
+
|
|
139
|
+
const config: PalCrawlConfig = {
|
|
140
|
+
userAgent: 'MyBot/1.0 (https://example.com)',
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
const palCrawl = new PalCrawl(config);
|
|
144
|
+
const data = await palCrawl.get();
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### 고급 설정 (타임아웃, 재시도, 헤더)
|
|
148
|
+
|
|
149
|
+
```typescript
|
|
150
|
+
import { PalCrawl, type PalCrawlConfig } from 'pal-crawl';
|
|
151
|
+
|
|
152
|
+
const config: PalCrawlConfig = {
|
|
153
|
+
userAgent: 'Advanced Bot 2.0',
|
|
154
|
+
timeout: 20000, // 20초 타임아웃
|
|
155
|
+
retryCount: 2, // 2회 재시도
|
|
156
|
+
customHeaders: {
|
|
157
|
+
'Accept-Language': 'ko-KR',
|
|
158
|
+
'Cache-Control': 'no-cache',
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
const palCrawl = new PalCrawl(config);
|
|
163
|
+
const data = await palCrawl.get();
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
### 에러 처리
|
|
167
|
+
|
|
168
|
+
```typescript
|
|
169
|
+
import { PalCrawl } from 'pal-crawl';
|
|
170
|
+
|
|
171
|
+
const palCrawl = new PalCrawl({
|
|
172
|
+
timeout: 5000,
|
|
173
|
+
retryCount: 3,
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
const data = await palCrawl.get();
|
|
178
|
+
console.log('입법예고 데이터:', data);
|
|
179
|
+
} catch (error) {
|
|
180
|
+
if (error.message.includes('timeout')) {
|
|
181
|
+
console.error('요청 타임아웃:', error.message);
|
|
182
|
+
} else {
|
|
183
|
+
console.error('크롤링 실패:', error.message);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
---
|
|
189
|
+
|
|
75
190
|
## License
|
|
76
191
|
|
|
77
192
|
This project is licensed under the MIT License. See the [LICENSE](#license) section for details.
|
package/dist/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { PalCrawl, type ITableData } from './pal';
|
|
1
|
+
export { PalCrawl, type ITableData, type IAttachment, type PalCrawlConfig, } from './pal';
|
|
2
2
|
export { Config } from './config';
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,6BAKe;AAJb,+FAAA,QAAQ,OAAA;AAKV,mCAAkC;AAAzB,gGAAA,MAAM,OAAA"}
|
package/dist/pal.d.ts
CHANGED
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
export interface PalCrawlConfig {
|
|
2
|
+
userAgent?: string;
|
|
3
|
+
timeout?: number;
|
|
4
|
+
retryCount?: number;
|
|
5
|
+
customHeaders?: Record<string, string>;
|
|
6
|
+
}
|
|
7
|
+
export interface IAttachment {
|
|
8
|
+
pdfFile: string;
|
|
9
|
+
hwpFile: string;
|
|
10
|
+
}
|
|
1
11
|
export interface ITableData {
|
|
2
12
|
num: number;
|
|
3
13
|
subject: string;
|
|
@@ -5,8 +15,15 @@ export interface ITableData {
|
|
|
5
15
|
committee: string;
|
|
6
16
|
numComments: number;
|
|
7
17
|
link: string;
|
|
18
|
+
attachments: IAttachment;
|
|
8
19
|
}
|
|
9
20
|
export declare class PalCrawl {
|
|
21
|
+
private userAgent;
|
|
22
|
+
private timeout;
|
|
23
|
+
private retryCount;
|
|
24
|
+
private customHeaders;
|
|
25
|
+
constructor(config?: PalCrawlConfig);
|
|
26
|
+
private makeRequest;
|
|
10
27
|
getPalHTML(): Promise<string>;
|
|
11
28
|
parseTable(html: string): ITableData[];
|
|
12
29
|
get(): Promise<ITableData[]>;
|
package/dist/pal.js
CHANGED
|
@@ -39,18 +39,25 @@ const url_1 = require("url");
|
|
|
39
39
|
const cheerio = __importStar(require("cheerio"));
|
|
40
40
|
const config_1 = require("./config");
|
|
41
41
|
class PalCrawl {
|
|
42
|
-
|
|
42
|
+
constructor(config) {
|
|
43
|
+
this.userAgent = config?.userAgent ?? config_1.Config.UserAgent;
|
|
44
|
+
this.timeout = config?.timeout ?? 10000; // Default 10 seconds
|
|
45
|
+
this.retryCount = config?.retryCount ?? 3; // Default 3 attempts
|
|
46
|
+
this.customHeaders = config?.customHeaders ?? {};
|
|
47
|
+
}
|
|
48
|
+
async makeRequest() {
|
|
43
49
|
return new Promise((resolve, reject) => {
|
|
44
50
|
const url = new url_1.URL(config_1.Config.URL, config_1.Config.DOMAIN);
|
|
51
|
+
const headers = {
|
|
52
|
+
'User-Agent': this.userAgent,
|
|
53
|
+
...this.customHeaders,
|
|
54
|
+
};
|
|
45
55
|
const options = {
|
|
46
|
-
headers
|
|
47
|
-
|
|
48
|
-
},
|
|
56
|
+
headers,
|
|
57
|
+
timeout: this.timeout,
|
|
49
58
|
};
|
|
50
|
-
https
|
|
51
|
-
.
|
|
52
|
-
if (res.statusCode &&
|
|
53
|
-
(res.statusCode < 200 || res.statusCode >= 300)) {
|
|
59
|
+
const req = https.get(url, options, (res) => {
|
|
60
|
+
if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) {
|
|
54
61
|
reject(new Error(`Invalid response: ${res.statusCode} ${res.statusMessage}`));
|
|
55
62
|
return;
|
|
56
63
|
}
|
|
@@ -61,12 +68,34 @@ class PalCrawl {
|
|
|
61
68
|
res.on('end', () => {
|
|
62
69
|
resolve(data);
|
|
63
70
|
});
|
|
64
|
-
})
|
|
65
|
-
|
|
71
|
+
});
|
|
72
|
+
req.on('timeout', () => {
|
|
73
|
+
req.destroy();
|
|
74
|
+
reject(new Error(`Request timeout after ${this.timeout}ms`));
|
|
75
|
+
});
|
|
76
|
+
req.on('error', (err) => {
|
|
66
77
|
reject(err);
|
|
67
78
|
});
|
|
79
|
+
req.setTimeout(this.timeout);
|
|
68
80
|
});
|
|
69
81
|
}
|
|
82
|
+
async getPalHTML() {
|
|
83
|
+
let lastError;
|
|
84
|
+
for (let attempt = 1; attempt <= this.retryCount; attempt++) {
|
|
85
|
+
try {
|
|
86
|
+
return await this.makeRequest();
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
lastError = error;
|
|
90
|
+
if (attempt < this.retryCount) {
|
|
91
|
+
// Exponential backoff before retrying
|
|
92
|
+
const delay = Math.min(1000 * Math.pow(2, attempt - 1), 5000);
|
|
93
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
throw lastError;
|
|
98
|
+
}
|
|
70
99
|
parseTable(html) {
|
|
71
100
|
const $ = cheerio.load(html);
|
|
72
101
|
const body = $('body');
|
|
@@ -84,6 +113,16 @@ class PalCrawl {
|
|
|
84
113
|
if (link) {
|
|
85
114
|
boardLink = config_1.Config.DOMAIN + link;
|
|
86
115
|
}
|
|
116
|
+
const pdfFile = $(el)
|
|
117
|
+
.find('td:nth-child(6) > a:nth-child(3)')
|
|
118
|
+
.attr('href');
|
|
119
|
+
const hwpFile = $(el)
|
|
120
|
+
.find('td:nth-child(6) > a:nth-child(2)')
|
|
121
|
+
.attr('href');
|
|
122
|
+
const attachments = {
|
|
123
|
+
pdfFile,
|
|
124
|
+
hwpFile,
|
|
125
|
+
};
|
|
87
126
|
output.push({
|
|
88
127
|
num,
|
|
89
128
|
subject,
|
|
@@ -91,6 +130,7 @@ class PalCrawl {
|
|
|
91
130
|
committee,
|
|
92
131
|
numComments,
|
|
93
132
|
link: boardLink,
|
|
133
|
+
attachments,
|
|
94
134
|
});
|
|
95
135
|
}
|
|
96
136
|
});
|
package/dist/pal.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pal.js","sourceRoot":"","sources":["../src/pal.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA+B;AAC/B,6BAA0B;AAC1B,iDAAmC;AACnC,qCAAkC;
|
|
1
|
+
{"version":3,"file":"pal.js","sourceRoot":"","sources":["../src/pal.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA+B;AAC/B,6BAA0B;AAC1B,iDAAmC;AACnC,qCAAkC;AAwBlC,MAAa,QAAQ;IAMnB,YAAY,MAAuB;QACjC,IAAI,CAAC,SAAS,GAAG,MAAM,EAAE,SAAS,IAAI,eAAM,CAAC,SAAS,CAAC;QACvD,IAAI,CAAC,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,KAAK,CAAC,CAAC,qBAAqB;QAC9D,IAAI,CAAC,UAAU,GAAG,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC,CAAC,qBAAqB;QAChE,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC;IACnD,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,eAAM,CAAC,GAAG,EAAE,eAAM,CAAC,MAAM,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAG;gBACd,YAAY,EAAE,IAAI,CAAC,SAAS;gBAC5B,GAAG,IAAI,CAAC,aAAa;aACtB,CAAC;YAEF,MAAM,OAAO,GAAG;gBACd,OAAO;gBACP,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC;YAEF,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC1C,IAAI,GAAG,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,CAAC;oBACtE,MAAM,CACJ,IAAI,KAAK,CACP,qBAAqB,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,aAAa,EAAE,CAC3D,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBAED,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;oBACvB,IAAI,IAAI,KAAK,CAAC;gBAChB,CAAC,CAAC,CAAC;gBAEH,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACjB,OAAO,CAAC,IAAI,CAAC,CAAC;gBAChB,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;gBACrB,GAAG,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACtB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,UAAU;QACrB,IAAI,SAAgB,CAAC;QAErB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YAClC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAc,CAAC;gBAE3B,IAAI,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;oBAC9B,sCAAsC;oBACtC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;oBAC9D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,SAAU,CAAC;IACnB,CAAC;IAEM,UAAU,CAAC,IAAY;QAC5B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAClB,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC1E,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtE,MAAM,WAAW,GAAG,MAAM,CACxB,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAC7D,CAAC;gBACF,MAAM,gBAAgB,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACrE,MAAM,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAC9D,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBAChE,IAAI,SAAS,GAAG,EAAE,CAAC;gBACnB,IAAI,IAAI,EAAE,CAAC;oBACT,SAAS,GAAG,eAAM,CAAC,MAAM,GAAG,IAAI,CAAC;gBACnC,CAAC;gBACD,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC;qBAClB,IAAI,CAAC,kCAAkC,CAAC;qBACxC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChB,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC;qBAClB,IAAI,CAAC,kCAAkC,CAAC;qBACxC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChB,MAAM,WAAW,GAAgB;oBAC/B,OAAO;oBACP,OAAO;iBACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC;oBACV,GAAG;oBACH,OAAO;oBACP,gBAAgB;oBAChB,SAAS;oBACT,WAAW;oBACX,IAAI,EAAE,SAAS;oBACf,WAAW;iBACZ,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,GAAG;QACd,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AA/HD,4BA+HC"}
|
package/dist/pal.test.js
CHANGED
|
@@ -1,21 +1,81 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const pal_1 = require("./pal");
|
|
4
|
+
let table;
|
|
4
5
|
describe('PalCrawl', () => {
|
|
6
|
+
beforeAll(async () => {
|
|
7
|
+
const palCrawl = new pal_1.PalCrawl();
|
|
8
|
+
table = await palCrawl.get();
|
|
9
|
+
});
|
|
5
10
|
test('getPalHTML: should be return string', async () => {
|
|
6
11
|
const palCrawl = new pal_1.PalCrawl();
|
|
7
12
|
const html = await palCrawl.getPalHTML();
|
|
8
13
|
expect(typeof html).toBe('string');
|
|
9
14
|
});
|
|
15
|
+
test('constructor: should use custom userAgent when provided in config', async () => {
|
|
16
|
+
const config = {
|
|
17
|
+
userAgent: 'Custom User Agent',
|
|
18
|
+
};
|
|
19
|
+
const palCrawl = new pal_1.PalCrawl(config);
|
|
20
|
+
expect(palCrawl).toBeInstanceOf(pal_1.PalCrawl);
|
|
21
|
+
});
|
|
22
|
+
test('constructor: should use default userAgent when config is empty', async () => {
|
|
23
|
+
const palCrawl = new pal_1.PalCrawl({});
|
|
24
|
+
expect(palCrawl).toBeInstanceOf(pal_1.PalCrawl);
|
|
25
|
+
});
|
|
26
|
+
test('constructor: should use default userAgent when no config provided', async () => {
|
|
27
|
+
const palCrawl = new pal_1.PalCrawl();
|
|
28
|
+
expect(palCrawl).toBeInstanceOf(pal_1.PalCrawl);
|
|
29
|
+
});
|
|
10
30
|
test('parseTable: array length should be 0 when invalid html', () => {
|
|
11
31
|
const palCrawl = new pal_1.PalCrawl();
|
|
12
32
|
const table = palCrawl.parseTable('asdf');
|
|
13
33
|
expect(table).toHaveLength(0);
|
|
14
34
|
});
|
|
15
35
|
test('get: array length should be 10', async () => {
|
|
16
|
-
const palCrawl = new pal_1.PalCrawl();
|
|
17
|
-
const table = await palCrawl.get();
|
|
18
36
|
expect(table).toHaveLength(10);
|
|
19
37
|
});
|
|
38
|
+
test('get: URL string of the attachment object should not be empty', () => {
|
|
39
|
+
table.forEach((e) => {
|
|
40
|
+
expect(e.attachments.pdfFile).not.toBe('');
|
|
41
|
+
expect(e.attachments.hwpFile).not.toBe('');
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
test('constructor: should use custom timeout when provided', () => {
|
|
45
|
+
const config = {
|
|
46
|
+
timeout: 5000,
|
|
47
|
+
};
|
|
48
|
+
const palCrawl = new pal_1.PalCrawl(config);
|
|
49
|
+
expect(palCrawl).toBeInstanceOf(pal_1.PalCrawl);
|
|
50
|
+
});
|
|
51
|
+
test('constructor: should use custom retryCount when provided', () => {
|
|
52
|
+
const config = {
|
|
53
|
+
retryCount: 5,
|
|
54
|
+
};
|
|
55
|
+
const palCrawl = new pal_1.PalCrawl(config);
|
|
56
|
+
expect(palCrawl).toBeInstanceOf(pal_1.PalCrawl);
|
|
57
|
+
});
|
|
58
|
+
test('constructor: should use custom headers when provided', () => {
|
|
59
|
+
const config = {
|
|
60
|
+
customHeaders: {
|
|
61
|
+
'Accept-Language': 'ko-KR',
|
|
62
|
+
'Custom-Header': 'test-value',
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
const palCrawl = new pal_1.PalCrawl(config);
|
|
66
|
+
expect(palCrawl).toBeInstanceOf(pal_1.PalCrawl);
|
|
67
|
+
});
|
|
68
|
+
test('constructor: should use all custom config options', () => {
|
|
69
|
+
const config = {
|
|
70
|
+
userAgent: 'Custom Agent',
|
|
71
|
+
timeout: 15000,
|
|
72
|
+
retryCount: 2,
|
|
73
|
+
customHeaders: {
|
|
74
|
+
Accept: 'application/json',
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
const palCrawl = new pal_1.PalCrawl(config);
|
|
78
|
+
expect(palCrawl).toBeInstanceOf(pal_1.PalCrawl);
|
|
79
|
+
});
|
|
20
80
|
});
|
|
21
81
|
//# sourceMappingURL=pal.test.js.map
|
package/dist/pal.test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pal.test.js","sourceRoot":"","sources":["../src/pal.test.ts"],"names":[],"mappings":";;AAAA,+
|
|
1
|
+
{"version":3,"file":"pal.test.js","sourceRoot":"","sources":["../src/pal.test.ts"],"names":[],"mappings":";;AAAA,+BAAkE;AAElE,IAAI,KAAmB,CAAC;AAExB,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;IACxB,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,QAAQ,GAAG,IAAI,cAAQ,EAAE,CAAC;QAChC,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,QAAQ,GAAG,IAAI,cAAQ,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;QACzC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,MAAM,GAAmB;YAC7B,SAAS,EAAE,mBAAmB;SAC/B,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QAChF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,QAAQ,GAAG,IAAI,cAAQ,EAAE,CAAC;QAChC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAClE,MAAM,QAAQ,GAAG,IAAI,cAAQ,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACxE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YAClB,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAChE,MAAM,MAAM,GAAmB;YAC7B,OAAO,EAAE,IAAI;SACd,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACnE,MAAM,MAAM,GAAmB;YAC7B,UAAU,EAAE,CAAC;SACd,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAChE,MAAM,MAAM,GAAmB;YAC7B,aAAa,EAAE;gBACb,iBAAiB,EAAE,OAAO;gBAC1B,eAAe,EAAE,YAAY;aAC9B;SACF,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC7D,MAAM,MAAM,GAAmB;YAC7B,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;YACd,UAAU,EAAE,CAAC;YACb,aAAa,EAAE;gBACb,MAAM,EAAE,kBAAkB;aAC3B;SACF,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|