pal-crawl 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -2
- package/dist/config.d.ts +2 -1
- package/dist/config.js +2 -1
- package/dist/config.js.map +1 -1
- package/dist/http-client.d.ts +19 -0
- package/dist/http-client.js +155 -0
- package/dist/http-client.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js.map +1 -1
- package/dist/pal.d.ts +13 -7
- package/dist/pal.js +77 -59
- package/dist/pal.js.map +1 -1
- package/dist/pal.test.js +68 -6
- package/dist/pal.test.js.map +1 -1
- package/package.json +5 -4
- package/tsconfig.json +1 -0
package/README.md
CHANGED
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
- [Base Types](#base-types)
|
|
22
22
|
- [Methods](#methods)
|
|
23
23
|
- [get](#get)
|
|
24
|
+
- [getContentHTML](#getcontenthtml--promisestring)
|
|
25
|
+
- [getContent](#getcontentid-string--promiseicontentdata)
|
|
24
26
|
- [Examples](#examples)
|
|
25
27
|
- [License](#license)
|
|
26
28
|
|
|
@@ -85,6 +87,7 @@ interface ITableData {
|
|
|
85
87
|
committee: string; // 소관 위원회
|
|
86
88
|
numComments: number; // 의견 수
|
|
87
89
|
link: string; // 전문 보기 링크
|
|
90
|
+
contentId: string | null; // 링크에서 추출한 본문 조회용 의안 ID
|
|
88
91
|
attachments: IAttachment; // 법률안 전문 첨부파일 URL 객체
|
|
89
92
|
}
|
|
90
93
|
```
|
|
@@ -95,8 +98,17 @@ pdf와 hwp 파일 다운로드 링크 추출을 지원합니다.
|
|
|
95
98
|
|
|
96
99
|
```typescript
|
|
97
100
|
interface IAttachment {
|
|
98
|
-
pdfFile: string;
|
|
99
|
-
hwpFile: string;
|
|
101
|
+
pdfFile: string | null;
|
|
102
|
+
hwpFile: string | null;
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`IContentData`는 입법예고 본문 요약 데이터를 나타내는 인터페이스입니다.
|
|
107
|
+
|
|
108
|
+
```typescript
|
|
109
|
+
interface IContentData {
|
|
110
|
+
title: string; // 본문 페이지 제목
|
|
111
|
+
proposalReason: string | null; // "제안이유 및 주요내용" (한 줄 문자열)
|
|
100
112
|
}
|
|
101
113
|
```
|
|
102
114
|
|
|
@@ -108,6 +120,8 @@ interface IAttachment {
|
|
|
108
120
|
|
|
109
121
|
`get` 메서드는 진행 중인 입법 예고 데이터를 가져옵니다.
|
|
110
122
|
|
|
123
|
+
반환되는 각 항목에는 `contentId`가 포함되며, 이 값을 `getContentHTML` 또는 `getContent`에 전달해 본문 조회를 바로 수행할 수 있습니다.
|
|
124
|
+
|
|
111
125
|
```typescript
|
|
112
126
|
import { PalCrawl } from 'pal-crawl';
|
|
113
127
|
|
|
@@ -117,6 +131,40 @@ const table = await palCrawl.get();
|
|
|
117
131
|
console.log(table);
|
|
118
132
|
```
|
|
119
133
|
|
|
134
|
+
### getContentHTML(id: string) => Promise<string>
|
|
135
|
+
|
|
136
|
+
`getContentHTML` 메서드는 특정 의안 ID의 본문 페이지 HTML 원문을 가져옵니다.
|
|
137
|
+
|
|
138
|
+
```typescript
|
|
139
|
+
import { PalCrawl } from 'pal-crawl';
|
|
140
|
+
|
|
141
|
+
const palCrawl = new PalCrawl();
|
|
142
|
+
const table = await palCrawl.get();
|
|
143
|
+
|
|
144
|
+
const id = table[0]?.contentId;
|
|
145
|
+
if (id) {
|
|
146
|
+
const contentHtml = await palCrawl.getContentHTML(id);
|
|
147
|
+
console.log(contentHtml);
|
|
148
|
+
}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### getContent(id: string) => Promise<IContentData>
|
|
152
|
+
|
|
153
|
+
`getContent` 메서드는 특정 의안 ID의 본문 페이지를 파싱해 JSON 객체(`IContentData`)로 반환합니다.
|
|
154
|
+
|
|
155
|
+
```typescript
|
|
156
|
+
import { PalCrawl } from 'pal-crawl';
|
|
157
|
+
|
|
158
|
+
const palCrawl = new PalCrawl();
|
|
159
|
+
const content = await palCrawl.getContent('PRC_W2W6V0D4D0B9C1B4B4Z6V2W0U7V2T9');
|
|
160
|
+
|
|
161
|
+
console.log(content);
|
|
162
|
+
// {
|
|
163
|
+
// title: '[2218288] 조세특례제한법 일부개정법률안(윤한홍의원 등 10인)',
|
|
164
|
+
// proposalReason: '...'
|
|
165
|
+
// }
|
|
166
|
+
```
|
|
167
|
+
|
|
120
168
|
---
|
|
121
169
|
|
|
122
170
|
## Examples
|
|
@@ -131,6 +179,22 @@ const data = await palCrawl.get();
|
|
|
131
179
|
console.log(data);
|
|
132
180
|
```
|
|
133
181
|
|
|
182
|
+
### 목록에서 contentId를 이용해 본문 가져오기
|
|
183
|
+
|
|
184
|
+
```typescript
|
|
185
|
+
import { PalCrawl } from 'pal-crawl';
|
|
186
|
+
|
|
187
|
+
const palCrawl = new PalCrawl();
|
|
188
|
+
const table = await palCrawl.get();
|
|
189
|
+
|
|
190
|
+
const first = table.find((item) => item.contentId);
|
|
191
|
+
if (first?.contentId) {
|
|
192
|
+
const content = await palCrawl.getContent(first.contentId);
|
|
193
|
+
console.log(content.title);
|
|
194
|
+
console.log(content.proposalReason);
|
|
195
|
+
}
|
|
196
|
+
```
|
|
197
|
+
|
|
134
198
|
### 사용자 정의 User-Agent 사용
|
|
135
199
|
|
|
136
200
|
```typescript
|
package/dist/config.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export declare enum Config {
|
|
2
2
|
DOMAIN = "https://pal.assembly.go.kr",
|
|
3
|
-
|
|
3
|
+
LIST_URL = "/napal/lgsltpa/lgsltpaOngoing/list.do",
|
|
4
|
+
CONTENT_URL = "/napal/lgsltpa/lgsltpaOngoing/view.do",
|
|
4
5
|
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
|
5
6
|
}
|
package/dist/config.js
CHANGED
|
@@ -4,7 +4,8 @@ exports.Config = void 0;
|
|
|
4
4
|
var Config;
|
|
5
5
|
(function (Config) {
|
|
6
6
|
Config["DOMAIN"] = "https://pal.assembly.go.kr";
|
|
7
|
-
Config["
|
|
7
|
+
Config["LIST_URL"] = "/napal/lgsltpa/lgsltpaOngoing/list.do";
|
|
8
|
+
Config["CONTENT_URL"] = "/napal/lgsltpa/lgsltpaOngoing/view.do";
|
|
8
9
|
Config["UserAgent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
9
10
|
})(Config || (exports.Config = Config = {}));
|
|
10
11
|
//# sourceMappingURL=config.js.map
|
package/dist/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;;AAAA,IAAY,
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;;AAAA,IAAY,MAKX;AALD,WAAY,MAAM;IAChB,+CAAqC,CAAA;IACrC,4DAAkD,CAAA;IAClD,+DAAqD,CAAA;IACrD,uIAA6H,CAAA;AAC/H,CAAC,EALW,MAAM,sBAAN,MAAM,QAKjB"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { URL } from 'url';
|
|
2
|
+
export interface HttpClientConfig {
|
|
3
|
+
userAgent: string;
|
|
4
|
+
timeout: number;
|
|
5
|
+
retryCount: number;
|
|
6
|
+
customHeaders: Record<string, string>;
|
|
7
|
+
}
|
|
8
|
+
export declare class HttpClient {
|
|
9
|
+
private userAgent;
|
|
10
|
+
private timeout;
|
|
11
|
+
private retryCount;
|
|
12
|
+
private customHeaders;
|
|
13
|
+
constructor(config: HttpClientConfig);
|
|
14
|
+
private extractCharset;
|
|
15
|
+
private normalizeCharset;
|
|
16
|
+
private decodeBody;
|
|
17
|
+
private makeRequest;
|
|
18
|
+
get(url: URL): Promise<string>;
|
|
19
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.HttpClient = void 0;
|
|
40
|
+
const https = __importStar(require("https"));
|
|
41
|
+
const iconv_lite_1 = __importDefault(require("iconv-lite"));
|
|
42
|
+
class HttpClient {
|
|
43
|
+
constructor(config) {
|
|
44
|
+
this.userAgent = config.userAgent;
|
|
45
|
+
this.timeout = config.timeout;
|
|
46
|
+
this.retryCount = config.retryCount;
|
|
47
|
+
this.customHeaders = config.customHeaders;
|
|
48
|
+
}
|
|
49
|
+
extractCharset(contentType, body) {
|
|
50
|
+
const contentTypeValue = Array.isArray(contentType)
|
|
51
|
+
? contentType.join(';')
|
|
52
|
+
: (contentType ?? '');
|
|
53
|
+
const headerMatch = contentTypeValue.match(/charset\s*=\s*([^;\s]+)/i);
|
|
54
|
+
if (headerMatch?.[1]) {
|
|
55
|
+
return headerMatch[1].trim().toLowerCase();
|
|
56
|
+
}
|
|
57
|
+
const head = body.subarray(0, 4096).toString('ascii');
|
|
58
|
+
const metaMatch = head.match(/<meta[^>]+charset=["']?([a-zA-Z0-9_-]+)/i);
|
|
59
|
+
if (metaMatch?.[1]) {
|
|
60
|
+
return metaMatch[1].trim().toLowerCase();
|
|
61
|
+
}
|
|
62
|
+
return 'utf-8';
|
|
63
|
+
}
|
|
64
|
+
normalizeCharset(charset) {
|
|
65
|
+
if (charset === 'ks_c_5601-1987' || charset === 'x-windows-949') {
|
|
66
|
+
return 'cp949';
|
|
67
|
+
}
|
|
68
|
+
if (charset === 'euc_kr') {
|
|
69
|
+
return 'euc-kr';
|
|
70
|
+
}
|
|
71
|
+
return charset;
|
|
72
|
+
}
|
|
73
|
+
decodeBody(body, contentType) {
|
|
74
|
+
const detected = this.normalizeCharset(this.extractCharset(contentType, body));
|
|
75
|
+
if (detected === 'utf-8' || detected === 'utf8') {
|
|
76
|
+
const utf8Text = body.toString('utf8');
|
|
77
|
+
// Some pages are served in legacy Korean encodings despite utf-8 hints.
|
|
78
|
+
// If replacement characters appear, try Korean legacy decoders and prefer cleaner output.
|
|
79
|
+
if (utf8Text.includes('�')) {
|
|
80
|
+
const fallbackEncodings = ['cp949', 'euc-kr'];
|
|
81
|
+
let bestText = utf8Text;
|
|
82
|
+
let bestScore = (utf8Text.match(/�/g) ?? []).length;
|
|
83
|
+
for (const encoding of fallbackEncodings) {
|
|
84
|
+
if (!iconv_lite_1.default.encodingExists(encoding)) {
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
const candidate = iconv_lite_1.default.decode(body, encoding);
|
|
88
|
+
const score = (candidate.match(/�/g) ?? []).length;
|
|
89
|
+
if (score < bestScore) {
|
|
90
|
+
bestText = candidate;
|
|
91
|
+
bestScore = score;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return bestText;
|
|
95
|
+
}
|
|
96
|
+
return utf8Text;
|
|
97
|
+
}
|
|
98
|
+
if (iconv_lite_1.default.encodingExists(detected)) {
|
|
99
|
+
return iconv_lite_1.default.decode(body, detected);
|
|
100
|
+
}
|
|
101
|
+
return body.toString('utf8');
|
|
102
|
+
}
|
|
103
|
+
async makeRequest(url) {
|
|
104
|
+
return new Promise((resolve, reject) => {
|
|
105
|
+
const headers = {
|
|
106
|
+
'User-Agent': this.userAgent,
|
|
107
|
+
...this.customHeaders,
|
|
108
|
+
};
|
|
109
|
+
const options = {
|
|
110
|
+
headers,
|
|
111
|
+
timeout: this.timeout,
|
|
112
|
+
};
|
|
113
|
+
const req = https.get(url, options, (res) => {
|
|
114
|
+
if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) {
|
|
115
|
+
reject(new Error(`Invalid response: ${res.statusCode} ${res.statusMessage}`));
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
const chunks = [];
|
|
119
|
+
res.on('data', (chunk) => {
|
|
120
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
121
|
+
});
|
|
122
|
+
res.on('end', () => {
|
|
123
|
+
const body = Buffer.concat(chunks);
|
|
124
|
+
resolve(this.decodeBody(body, res.headers['content-type']));
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
req.on('timeout', () => {
|
|
128
|
+
req.destroy();
|
|
129
|
+
reject(new Error(`Request timeout after ${this.timeout}ms`));
|
|
130
|
+
});
|
|
131
|
+
req.on('error', (err) => {
|
|
132
|
+
reject(err);
|
|
133
|
+
});
|
|
134
|
+
req.setTimeout(this.timeout);
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
async get(url) {
|
|
138
|
+
let lastError;
|
|
139
|
+
for (let attempt = 1; attempt <= this.retryCount; attempt++) {
|
|
140
|
+
try {
|
|
141
|
+
return await this.makeRequest(url);
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
lastError = error;
|
|
145
|
+
if (attempt < this.retryCount) {
|
|
146
|
+
const delay = Math.min(1000 * Math.pow(2, attempt - 1), 5000);
|
|
147
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
throw lastError;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
exports.HttpClient = HttpClient;
|
|
155
|
+
//# sourceMappingURL=http-client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../src/http-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA+B;AAE/B,4DAA+B;AAS/B,MAAa,UAAU;IAMrB,YAAY,MAAwB;QAClC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAC5C,CAAC;IAEO,cAAc,CACpB,WAA0C,EAC1C,IAAY;QAEZ,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC;YACjD,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC;YACvB,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAExB,MAAM,WAAW,GAAG,gBAAgB,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACvE,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrB,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;QACzE,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnB,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,gBAAgB,CAAC,OAAe;QACtC,IAAI,OAAO,KAAK,gBAAgB,IAAI,OAAO,KAAK,eAAe,EAAE,CAAC;YAChE,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzB,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,UAAU,CAChB,IAAY,EACZ,WAA0C;QAE1C,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CACpC,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,IAAI,CAAC,CACvC,CAAC;QAEF,IAAI,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;YAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAEvC,wEAAwE;YACxE,0FAA0F;YAC1F,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC9C,IAAI,QAAQ,GAAG,QAAQ,CAAC;gBACxB,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;gBAEpD,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;oBACzC,IAAI,CAAC,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACpC,SAAS;oBACX,CAAC;oBAED,MAAM,SAAS,GAAG,oBAAK,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;oBAC/C,MAAM,KAAK,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;oBACnD,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;wBACtB,QAAQ,GAAG,SAAS,CAAC;wBACrB,SAAS,GAAG,KAAK,CAAC;oBACpB,CAAC;gBACH,CAAC;gBAED,OAAO,QAAQ,CAAC;YAClB,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,IAAI,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnC,OAAO,oBAAK,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAQ;QAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAG;gBACd,YAAY,EAAE,IAAI,CAAC,SAAS;gBAC5B,GAAG,IAAI,CAAC,aAAa;aACtB,CAAC;YAEF,MAAM,OAAO,GAAG;gBACd,OAAO;gBACP,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC;YAEF,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC1C,IAAI,GAAG,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,CAAC;oBACtE,MAAM,CACJ,IAAI,KAAK,CACP,qBAAqB,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,aAAa,EAAE,CAC3D,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBAED,MAAM,MAAM,GAAa,EAAE,CAAC;gBAC5B,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;oBACvB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACnE,CAAC,CAAC,CAAC;gBAEH,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACjB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBACnC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;gBAC9D,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;gBACrB,GAAG,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACtB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,GAAQ;QACvB,IAAI,SAAgB,CAAC;QAErB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACrC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAc,CAAC;gBAE3B,IAAI,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;oBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;oBAC9D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,SAAU,CAAC;IACnB,CAAC;CACF;AAzJD,gCAyJC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { PalCrawl, type ITableData, type IAttachment, type PalCrawlConfig, } from './pal';
|
|
1
|
+
export { PalCrawl, type ITableData, type IAttachment, type IContentData, type PalCrawlConfig, } from './pal';
|
|
2
2
|
export { Config } from './config';
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,6BAMe;AALb,+FAAA,QAAQ,OAAA;AAMV,mCAAkC;AAAzB,gGAAA,MAAM,OAAA"}
|
package/dist/pal.d.ts
CHANGED
|
@@ -5,8 +5,8 @@ export interface PalCrawlConfig {
|
|
|
5
5
|
customHeaders?: Record<string, string>;
|
|
6
6
|
}
|
|
7
7
|
export interface IAttachment {
|
|
8
|
-
pdfFile: string;
|
|
9
|
-
hwpFile: string;
|
|
8
|
+
pdfFile: string | null;
|
|
9
|
+
hwpFile: string | null;
|
|
10
10
|
}
|
|
11
11
|
export interface ITableData {
|
|
12
12
|
num: number;
|
|
@@ -15,16 +15,22 @@ export interface ITableData {
|
|
|
15
15
|
committee: string;
|
|
16
16
|
numComments: number;
|
|
17
17
|
link: string;
|
|
18
|
+
contentId: string | null;
|
|
18
19
|
attachments: IAttachment;
|
|
19
20
|
}
|
|
21
|
+
export interface IContentData {
|
|
22
|
+
title: string;
|
|
23
|
+
proposalReason: string | null;
|
|
24
|
+
}
|
|
20
25
|
export declare class PalCrawl {
|
|
21
|
-
private
|
|
22
|
-
private timeout;
|
|
23
|
-
private retryCount;
|
|
24
|
-
private customHeaders;
|
|
26
|
+
private readonly httpClient;
|
|
25
27
|
constructor(config?: PalCrawlConfig);
|
|
26
|
-
private makeRequest;
|
|
27
28
|
getPalHTML(): Promise<string>;
|
|
29
|
+
private extractContentId;
|
|
28
30
|
parseTable(html: string): ITableData[];
|
|
31
|
+
private normalizeText;
|
|
32
|
+
parseContent(html: string): IContentData;
|
|
29
33
|
get(): Promise<ITableData[]>;
|
|
34
|
+
getContentHTML(id: string): Promise<string>;
|
|
35
|
+
getContent(id: string): Promise<IContentData>;
|
|
30
36
|
}
|
package/dist/pal.js
CHANGED
|
@@ -34,74 +34,38 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.PalCrawl = void 0;
|
|
37
|
-
const https = __importStar(require("https"));
|
|
38
37
|
const url_1 = require("url");
|
|
39
38
|
const cheerio = __importStar(require("cheerio"));
|
|
40
39
|
const config_1 = require("./config");
|
|
40
|
+
const http_client_1 = require("./http-client");
|
|
41
41
|
class PalCrawl {
|
|
42
42
|
constructor(config) {
|
|
43
|
-
this.
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
async makeRequest() {
|
|
49
|
-
return new Promise((resolve, reject) => {
|
|
50
|
-
const url = new url_1.URL(config_1.Config.URL, config_1.Config.DOMAIN);
|
|
51
|
-
const headers = {
|
|
52
|
-
'User-Agent': this.userAgent,
|
|
53
|
-
...this.customHeaders,
|
|
54
|
-
};
|
|
55
|
-
const options = {
|
|
56
|
-
headers,
|
|
57
|
-
timeout: this.timeout,
|
|
58
|
-
};
|
|
59
|
-
const req = https.get(url, options, (res) => {
|
|
60
|
-
if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) {
|
|
61
|
-
reject(new Error(`Invalid response: ${res.statusCode} ${res.statusMessage}`));
|
|
62
|
-
return;
|
|
63
|
-
}
|
|
64
|
-
let data = '';
|
|
65
|
-
res.on('data', (chunk) => {
|
|
66
|
-
data += chunk;
|
|
67
|
-
});
|
|
68
|
-
res.on('end', () => {
|
|
69
|
-
resolve(data);
|
|
70
|
-
});
|
|
71
|
-
});
|
|
72
|
-
req.on('timeout', () => {
|
|
73
|
-
req.destroy();
|
|
74
|
-
reject(new Error(`Request timeout after ${this.timeout}ms`));
|
|
75
|
-
});
|
|
76
|
-
req.on('error', (err) => {
|
|
77
|
-
reject(err);
|
|
78
|
-
});
|
|
79
|
-
req.setTimeout(this.timeout);
|
|
43
|
+
this.httpClient = new http_client_1.HttpClient({
|
|
44
|
+
userAgent: config?.userAgent ?? config_1.Config.UserAgent,
|
|
45
|
+
timeout: config?.timeout ?? 10000,
|
|
46
|
+
retryCount: config?.retryCount ?? 3,
|
|
47
|
+
customHeaders: config?.customHeaders ?? {},
|
|
80
48
|
});
|
|
81
49
|
}
|
|
82
50
|
async getPalHTML() {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
94
|
-
}
|
|
95
|
-
}
|
|
51
|
+
const url = new url_1.URL(config_1.Config.LIST_URL, config_1.Config.DOMAIN);
|
|
52
|
+
return this.httpClient.get(url);
|
|
53
|
+
}
|
|
54
|
+
extractContentId(link) {
|
|
55
|
+
try {
|
|
56
|
+
const url = new url_1.URL(link, config_1.Config.DOMAIN);
|
|
57
|
+
return (url.searchParams.get('lgsltPaId') ?? url.searchParams.get('lgsltPaid'));
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return null;
|
|
96
61
|
}
|
|
97
|
-
throw lastError;
|
|
98
62
|
}
|
|
99
63
|
parseTable(html) {
|
|
100
64
|
const $ = cheerio.load(html);
|
|
101
65
|
const body = $('body');
|
|
102
66
|
const table = body.find('table > tbody > tr');
|
|
103
67
|
const output = [];
|
|
104
|
-
table.map((
|
|
68
|
+
table.map((_i, el) => {
|
|
105
69
|
const subject = $(el).find('td.td_block > a.board_subject').text().trim();
|
|
106
70
|
if (subject) {
|
|
107
71
|
const link = $(el).find('td.td_block > a.board_subject').attr('href');
|
|
@@ -113,16 +77,13 @@ class PalCrawl {
|
|
|
113
77
|
if (link) {
|
|
114
78
|
boardLink = config_1.Config.DOMAIN + link;
|
|
115
79
|
}
|
|
116
|
-
const pdfFile = $(el)
|
|
117
|
-
|
|
118
|
-
.attr('href');
|
|
119
|
-
const hwpFile = $(el)
|
|
120
|
-
.find('td:nth-child(6) > a:nth-child(2)')
|
|
121
|
-
.attr('href');
|
|
80
|
+
const pdfFile = $(el).find('td:nth-child(6) > a:nth-child(3)').attr('href') ?? null;
|
|
81
|
+
const hwpFile = $(el).find('td:nth-child(6) > a:nth-child(2)').attr('href') ?? null;
|
|
122
82
|
const attachments = {
|
|
123
83
|
pdfFile,
|
|
124
84
|
hwpFile,
|
|
125
85
|
};
|
|
86
|
+
const contentId = boardLink ? this.extractContentId(boardLink) : null;
|
|
126
87
|
output.push({
|
|
127
88
|
num,
|
|
128
89
|
subject,
|
|
@@ -130,17 +91,74 @@ class PalCrawl {
|
|
|
130
91
|
committee,
|
|
131
92
|
numComments,
|
|
132
93
|
link: boardLink,
|
|
94
|
+
contentId,
|
|
133
95
|
attachments,
|
|
134
96
|
});
|
|
135
97
|
}
|
|
136
98
|
});
|
|
137
99
|
return output;
|
|
138
100
|
}
|
|
101
|
+
normalizeText(text) {
|
|
102
|
+
return text
|
|
103
|
+
.replace(/\u00a0/g, ' ')
|
|
104
|
+
.replace(/\r/g, '')
|
|
105
|
+
.split('\n')
|
|
106
|
+
.map((line) => line.replace(/\s+/g, ' ').trim())
|
|
107
|
+
.filter(Boolean)
|
|
108
|
+
.join('\n');
|
|
109
|
+
}
|
|
110
|
+
parseContent(html) {
|
|
111
|
+
const $ = cheerio.load(html);
|
|
112
|
+
const title = this.normalizeText($('.legislation-heading h3').first().text() || $('h3').first().text());
|
|
113
|
+
const sections = {};
|
|
114
|
+
$('.card-wrap .item').each((_, item) => {
|
|
115
|
+
const heading = this.normalizeText($(item).find('h4').first().text());
|
|
116
|
+
if (!heading) {
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
const descText = this.normalizeText($(item).find('.desc').first().text());
|
|
120
|
+
if (!descText) {
|
|
121
|
+
sections[heading] = '';
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
const normalizedHeading = heading.replace(/\s+/g, '');
|
|
125
|
+
const lines = descText.split('\n').filter(Boolean);
|
|
126
|
+
const firstLine = lines[0]?.replace(/\s+/g, '') ?? '';
|
|
127
|
+
// The first line sometimes repeats the section heading itself.
|
|
128
|
+
const content = firstLine === normalizedHeading
|
|
129
|
+
? this.normalizeText(lines.slice(1).join('\n'))
|
|
130
|
+
: descText;
|
|
131
|
+
sections[heading] = content;
|
|
132
|
+
});
|
|
133
|
+
const proposalReasonRaw = Object.entries(sections).find(([heading]) => heading.replace(/\s+/g, '').includes('제안이유및주요내용'))?.[1] ?? null;
|
|
134
|
+
const proposalReason = proposalReasonRaw
|
|
135
|
+
? this.normalizeText(proposalReasonRaw)
|
|
136
|
+
: null;
|
|
137
|
+
return {
|
|
138
|
+
title,
|
|
139
|
+
proposalReason,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
139
142
|
async get() {
|
|
140
143
|
const html = await this.getPalHTML();
|
|
141
144
|
const table = this.parseTable(html);
|
|
142
145
|
return table;
|
|
143
146
|
}
|
|
147
|
+
async getContentHTML(id) {
|
|
148
|
+
const normalizedId = id.trim();
|
|
149
|
+
if (!normalizedId) {
|
|
150
|
+
throw new Error('id is required');
|
|
151
|
+
}
|
|
152
|
+
const url = new url_1.URL(config_1.Config.CONTENT_URL, config_1.Config.DOMAIN);
|
|
153
|
+
url.searchParams.set('lgsltPaid', normalizedId);
|
|
154
|
+
// Keep compatibility with currently used query key on the website.
|
|
155
|
+
url.searchParams.set('lgsltPaId', normalizedId);
|
|
156
|
+
return this.httpClient.get(url);
|
|
157
|
+
}
|
|
158
|
+
async getContent(id) {
|
|
159
|
+
const html = await this.getContentHTML(id);
|
|
160
|
+
return this.parseContent(html);
|
|
161
|
+
}
|
|
144
162
|
}
|
|
145
163
|
exports.PalCrawl = PalCrawl;
|
|
146
164
|
//# sourceMappingURL=pal.js.map
|
package/dist/pal.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pal.js","sourceRoot":"","sources":["../src/pal.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"pal.js","sourceRoot":"","sources":["../src/pal.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6BAA0B;AAC1B,iDAAmC;AACnC,qCAAkC;AAClC,+CAA2C;AA8B3C,MAAa,QAAQ;IAGnB,YAAY,MAAuB;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,wBAAU,CAAC;YAC/B,SAAS,EAAE,MAAM,EAAE,SAAS,IAAI,eAAM,CAAC,SAAS;YAChD,OAAO,EAAE,MAAM,EAAE,OAAO,IAAI,KAAK;YACjC,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,CAAC;YACnC,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,EAAE;SAC3C,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,UAAU;QACrB,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,eAAM,CAAC,QAAQ,EAAE,eAAM,CAAC,MAAM,CAAC,CAAC;QACpD,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAEO,gBAAgB,CAAC,IAAY;QACnC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,IAAI,EAAE,eAAM,CAAC,MAAM,CAAC,CAAC;YACzC,OAAO,CACL,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CACvE,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEM,UAAU,CAAC,IAAY;QAC5B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;YACnB,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC1E,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtE,MAAM,WAAW,GAAG,MAAM,CACxB,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAC7D,CAAC;gBACF,MAAM,gBAAgB,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACrE,MAAM,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAC9D,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBAChE,IAAI,SAAS,GAAG,EAAE,CAAC;gBACnB,IAAI,IAAI,EAAE,CAAC;oBACT,SAAS,GAAG,eAAM,CAAC,MAAM,GAAG,IAAI,CAAC;gBACnC,CAAC;gBACD,MAAM,OAAO,GACX,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;gBACtE,MAAM,OAAO,GACX,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;gBACtE,MAAM,WAAW,GAAgB;oBAC/B,OAAO;oBACP,OAAO;iBACR,CAAC;gBACF,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACtE,MAAM,CAAC,IAAI,CAAC;oBACV,GAAG;oBACH,OAAO;oBACP,gBAAgB;oBAChB,SAAS;oBACT,WAAW;oBACX,IAAI,EAAE,SAAS;oBACf,SAAS;oBACT,WAAW;iBACZ,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,aAAa,CAAC,IAAY;QAChC,OAAO,IAAI;aACR,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;aAClB,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;aAC/C,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEM,YAAY,CAAC,IAAY;QAC9B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAC9B,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CACtE,CAAC;QAEF,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAE5C,CAAC,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO;YACT,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1E,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBACvB,OAAO;YACT,CAAC;YAED,MAAM,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;YAEtD,+DAA+D;YAC/D,MAAM,OAAO,GACX,SAAS,KAAK,iBAAiB;gBAC7B,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/C,CAAC,CAAC,QAAQ,CAAC;YAEf,QAAQ,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,MAAM,iBAAiB,GACrB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAC1C,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAClD,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QAEjB,MAAM,cAAc,GAAG,iBAAiB;YACtC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC;YACvC,CAAC,CAAC,IAAI,CAAC;QAET,OAAO;YACL,KAAK;YACL,cAAc;SACf,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,GAAG;QACd,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,KAAK,CAAC;IACf,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,EAAU;QACpC,MAAM,YAAY,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,eAAM,CAAC,WAAW,EAAE,eAAM,CAAC,MAAM,CAAC,CAAC;QACvD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QAChD,mEAAmE;QACnE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,EAAU;QAChC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;CACF;AA1JD,4BA0JC"}
|
package/dist/pal.test.js
CHANGED
|
@@ -1,16 +1,33 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const pal_1 = require("./pal");
|
|
4
|
+
const FIXED_CONTENT_ID = 'PRC_W2W6V0D4D0B9C1B4B4Z6V2W0U7V2T9';
|
|
5
|
+
const SHOULD_DEBUG = process.env.PAL_CRAWL_TEST_DEBUG === '1';
|
|
6
|
+
const debugLog = (label, value) => {
|
|
7
|
+
if (!SHOULD_DEBUG) {
|
|
8
|
+
return;
|
|
9
|
+
}
|
|
10
|
+
console.debug(label, JSON.stringify(value, null, 2));
|
|
11
|
+
};
|
|
4
12
|
let table;
|
|
13
|
+
let listHtml = '';
|
|
14
|
+
let contentHtml = '';
|
|
15
|
+
let fixedContent;
|
|
5
16
|
describe('PalCrawl', () => {
|
|
17
|
+
const palCrawl = new pal_1.PalCrawl();
|
|
6
18
|
beforeAll(async () => {
|
|
7
|
-
|
|
8
|
-
|
|
19
|
+
[listHtml, contentHtml] = await Promise.all([
|
|
20
|
+
palCrawl.getPalHTML(),
|
|
21
|
+
palCrawl.getContentHTML(FIXED_CONTENT_ID),
|
|
22
|
+
]);
|
|
23
|
+
table = palCrawl.parseTable(listHtml);
|
|
24
|
+
fixedContent = palCrawl.parseContent(contentHtml);
|
|
25
|
+
debugLog('table', table);
|
|
26
|
+
debugLog('content', fixedContent);
|
|
9
27
|
});
|
|
10
|
-
test('getPalHTML: should be return string',
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
expect(typeof html).toBe('string');
|
|
28
|
+
test('getPalHTML: should be return string', () => {
|
|
29
|
+
expect(typeof listHtml).toBe('string');
|
|
30
|
+
expect(listHtml).not.toContain('�');
|
|
14
31
|
});
|
|
15
32
|
test('constructor: should use custom userAgent when provided in config', async () => {
|
|
16
33
|
const config = {
|
|
@@ -32,6 +49,46 @@ describe('PalCrawl', () => {
|
|
|
32
49
|
const table = palCrawl.parseTable('asdf');
|
|
33
50
|
expect(table).toHaveLength(0);
|
|
34
51
|
});
|
|
52
|
+
test('parseContent: should parse title, reason, and opinion submission method', () => {
|
|
53
|
+
const palCrawl = new pal_1.PalCrawl();
|
|
54
|
+
const html = `
|
|
55
|
+
<div class="legislation-heading">
|
|
56
|
+
<h3>[2218288] 조세특례제한법 일부개정법률안(윤한홍의원 등 10인)</h3>
|
|
57
|
+
</div>
|
|
58
|
+
<div class="card-wrap">
|
|
59
|
+
<div class="item">
|
|
60
|
+
<h4>제안이유 및 주요내용</h4>
|
|
61
|
+
<div class="desc">
|
|
62
|
+
제안이유 및 주요내용
|
|
63
|
+
<br/>첫 번째 문장
|
|
64
|
+
<br/>두 번째 문장
|
|
65
|
+
</div>
|
|
66
|
+
</div>
|
|
67
|
+
<div class="item">
|
|
68
|
+
<h4>의견제출 방법</h4>
|
|
69
|
+
<div class="desc">국회 재정경제기획위원회</div>
|
|
70
|
+
</div>
|
|
71
|
+
<div class="item item-means">
|
|
72
|
+
<div class="desc">제출방법: 상단 의견등록 탭 사용</div>
|
|
73
|
+
</div>
|
|
74
|
+
</div>
|
|
75
|
+
`;
|
|
76
|
+
const content = palCrawl.parseContent(html);
|
|
77
|
+
expect(content.title).toContain('조세특례제한법 일부개정법률안');
|
|
78
|
+
expect(content.proposalReason).toBe('첫 번째 문장\n두 번째 문장');
|
|
79
|
+
});
|
|
80
|
+
test('getContentHTML: should prefetch real content html by fixed id', () => {
|
|
81
|
+
expect(typeof contentHtml).toBe('string');
|
|
82
|
+
expect(contentHtml.length).toBeGreaterThan(0);
|
|
83
|
+
});
|
|
84
|
+
test('getContent: should parse prefetched content html by fixed id', () => {
|
|
85
|
+
const content = fixedContent;
|
|
86
|
+
expect(content.title.length).toBeGreaterThan(0);
|
|
87
|
+
expect(content.proposalReason).not.toBeNull();
|
|
88
|
+
expect(content.proposalReason?.length ?? 0).toBeGreaterThan(0);
|
|
89
|
+
expect(content.title).not.toContain('�');
|
|
90
|
+
expect(content.proposalReason ?? '').not.toContain('�');
|
|
91
|
+
});
|
|
35
92
|
test('get: array length should be 10', async () => {
|
|
36
93
|
expect(table).toHaveLength(10);
|
|
37
94
|
});
|
|
@@ -39,6 +96,11 @@ describe('PalCrawl', () => {
|
|
|
39
96
|
table.forEach((e) => {
|
|
40
97
|
expect(e.attachments.pdfFile).not.toBe('');
|
|
41
98
|
expect(e.attachments.hwpFile).not.toBe('');
|
|
99
|
+
expect(e.subject).not.toContain('�');
|
|
100
|
+
expect(e.proposerCategory).not.toContain('�');
|
|
101
|
+
expect(e.committee).not.toContain('�');
|
|
102
|
+
expect(e.contentId).not.toBeNull();
|
|
103
|
+
expect(e.contentId?.startsWith('PRC_')).toBe(true);
|
|
42
104
|
});
|
|
43
105
|
});
|
|
44
106
|
test('constructor: should use custom timeout when provided', () => {
|
package/dist/pal.test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pal.test.js","sourceRoot":"","sources":["../src/pal.test.ts"],"names":[],"mappings":";;AAAA,+BAAkE;AAElE,
|
|
1
|
+
{"version":3,"file":"pal.test.js","sourceRoot":"","sources":["../src/pal.test.ts"],"names":[],"mappings":";;AAAA,+BAAkE;AAElE,MAAM,gBAAgB,GAAG,oCAAoC,CAAC;AAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,GAAG,CAAC;AAE9D,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,KAAc,EAAQ,EAAE;IACvD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;IACT,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,IAAI,KAAmB,CAAC;AACxB,IAAI,QAAQ,GAAG,EAAE,CAAC;AAClB,IAAI,WAAW,GAAG,EAAE,CAAC;AACrB,IAAI,YAAyD,CAAC;AAE9D,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;IACxB,MAAM,QAAQ,GAAG,IAAI,cAAQ,EAAE,CAAC;IAEhC,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC1C,QAAQ,CAAC,UAAU,EAAE;YACrB,QAAQ,CAAC,cAAc,CAAC,gBAAgB,CAAC;SAC1C,CAAC,CAAC;QACH,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACtC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QAClD,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACzB,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC/C,MAAM,CAAC,OAAO,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,MAAM,GAAmB;YAC7B,SAAS,EAAE,mBAAmB;SAC/B,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QAChF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,QAAQ,GAAG,IAAI,cAAQ,EAAE,CAAC;QAChC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAClE,MAAM,QAAQ,GAAG,IAAI,cAAQ,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,yEAAyE,EAAE,GAAG,EAAE;QACnF,MAAM,QAAQ,GAAG,IAAI,cAAQ,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;KAqBZ,CAAC;QAEF,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAE5C,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QACnD,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACzE,MAAM,CAAC,OAAO,WAAW,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACxE,MAAM,OAAO,GAAG,YAAY,CAAC;QAE7B,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC9C,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACxE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YAClB,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC9C,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACnC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAChE,MAAM,MAAM,GAAmB;YAC7B,OAAO,EAAE,IAAI;SACd,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACnE,MAAM,MAAM,GAAmB;YAC7B,UAAU,EAAE,CAAC;SACd,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAChE,MAAM,MAAM,GAAmB;YAC7B,aAAa,EAAE;gBACb,iBAAiB,EAAE,OAAO;gBAC1B,eAAe,EAAE,YAAY;aAC9B;SACF,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC7D,MAAM,MAAM,GAAmB;YAC7B,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;YACd,UAAU,EAAE,CAAC;YACb,aAAa,EAAE;gBACb,MAAM,EAAE,kBAAkB;aAC3B;SACF,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,cAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,cAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pal-crawl",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "국회입법예고(pal.assembly.go.kr)의 진행 중인 입법 예고 크롤러",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "vientorepublic",
|
|
@@ -34,12 +34,13 @@
|
|
|
34
34
|
},
|
|
35
35
|
"homepage": "https://github.com/vientorepublic/pal-crawl#readme",
|
|
36
36
|
"dependencies": {
|
|
37
|
-
"cheerio": "^1.0.0"
|
|
37
|
+
"cheerio": "^1.0.0",
|
|
38
|
+
"iconv-lite": "^0.7.2"
|
|
38
39
|
},
|
|
39
40
|
"devDependencies": {
|
|
40
|
-
"@eslint/js": "^
|
|
41
|
+
"@eslint/js": "^10.0.0",
|
|
41
42
|
"@types/jest": "^30.0.0",
|
|
42
|
-
"eslint": "^
|
|
43
|
+
"eslint": "^10.0.0",
|
|
43
44
|
"jest": "^30.0.0",
|
|
44
45
|
"prettier": "^3.4.2",
|
|
45
46
|
"ts-jest": "^29.2.5",
|