pal-crawl 1.6.1 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +293 -4
- package/dist/config.d.ts +2 -0
- package/dist/config.js +2 -0
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/pal.d.ts +44 -27
- package/dist/pal.js +115 -174
- package/dist/pal.js.map +1 -1
- package/dist/pal.test.js +560 -171
- package/dist/pal.test.js.map +1 -1
- package/dist/parser.d.ts +40 -0
- package/dist/parser.js +210 -0
- package/dist/parser.js.map +1 -0
- package/package.json +1 -1
package/dist/pal.test.js
CHANGED
|
@@ -1,195 +1,584 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const pal_1 = require("./pal");
|
|
4
|
-
const
|
|
4
|
+
const parser_1 = require("./parser");
|
|
5
|
+
const FIXED_ONGOING_CONTENT_ID = 'PRC_W2W6V0D4D0B9C1B4B4Z6V2W0U7V2T9';
|
|
6
|
+
const FIXED_DONE_CONTENT_ID = 'PRC_S2R6N0M3K2J3K1J5K3S8R3P4O3P5X6';
|
|
5
7
|
const SHOULD_DEBUG = process.env.PAL_CRAWL_TEST_DEBUG === '1';
|
|
6
8
|
const debugLog = (label, value) => {
|
|
7
|
-
if (!SHOULD_DEBUG)
|
|
9
|
+
if (!SHOULD_DEBUG)
|
|
8
10
|
return;
|
|
9
|
-
}
|
|
10
11
|
console.debug(label, JSON.stringify(value, null, 2));
|
|
11
12
|
};
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
13
|
+
// ─── HTML Fixtures ────────────────────────────────────────────────────────────
|
|
14
|
+
const TABLE_ROW_HTML = `
|
|
15
|
+
<table>
|
|
16
|
+
<tbody>
|
|
17
|
+
<tr>
|
|
18
|
+
<td>42</td>
|
|
19
|
+
<td class="td_block">
|
|
20
|
+
<a href="/napal/lgsltpa/lgsltpaOngoing/view.do?lgsltPaId=PRC_TEST000" class="board_subject">테스트 법률안 (홍길동의원 등 5인)</a>
|
|
21
|
+
</td>
|
|
22
|
+
<td>의원</td>
|
|
23
|
+
<td>법제사법위원회</td>
|
|
24
|
+
<td></td>
|
|
25
|
+
<td>
|
|
26
|
+
<a href="#">버튼</a>
|
|
27
|
+
<a href="https://example.com/file.hwp">hwp</a>
|
|
28
|
+
<a href="https://example.com/file.pdf">pdf</a>
|
|
29
|
+
</td>
|
|
30
|
+
<td></td>
|
|
31
|
+
<td>1,234</td>
|
|
32
|
+
<td></td>
|
|
33
|
+
</tr>
|
|
34
|
+
</tbody>
|
|
35
|
+
</table>
|
|
36
|
+
`;
|
|
37
|
+
const TABLE_ROW_NO_LINK_HTML = `
|
|
38
|
+
<table>
|
|
39
|
+
<tbody>
|
|
40
|
+
<tr>
|
|
41
|
+
<td>1</td>
|
|
42
|
+
<td class="td_block">
|
|
43
|
+
<a class="board_subject">링크없는 법률안</a>
|
|
44
|
+
</td>
|
|
45
|
+
<td>정부</td>
|
|
46
|
+
<td>기획재정위원회</td>
|
|
47
|
+
<td></td>
|
|
48
|
+
<td></td>
|
|
49
|
+
<td></td>
|
|
50
|
+
<td>0</td>
|
|
51
|
+
<td></td>
|
|
52
|
+
</tr>
|
|
53
|
+
</tbody>
|
|
54
|
+
</table>
|
|
55
|
+
`;
|
|
56
|
+
const CONTENT_HTML = `
|
|
57
|
+
<div class="legislation-heading">
|
|
58
|
+
<h3>[2218288] 조세특례제한법 일부개정법률안(윤한홍의원 등 10인)</h3>
|
|
59
|
+
</div>
|
|
60
|
+
<div class="board01 pr td_center board-added">
|
|
61
|
+
<table>
|
|
62
|
+
<thead>
|
|
63
|
+
<tr>
|
|
64
|
+
<th scope="col">의안번호</th>
|
|
65
|
+
<th scope="col">제안자</th>
|
|
66
|
+
<th scope="col">제안일</th>
|
|
67
|
+
<th scope="col">소관위원회</th>
|
|
68
|
+
<th scope="col">회부일</th>
|
|
69
|
+
<th scope="col">입법예고기간</th>
|
|
70
|
+
<th scope="col">법률안원문</th>
|
|
71
|
+
<th scope="col">제안회기</th>
|
|
72
|
+
</tr>
|
|
73
|
+
</thead>
|
|
74
|
+
<tbody>
|
|
75
|
+
<tr>
|
|
76
|
+
<td>2218288</td>
|
|
77
|
+
<td>
|
|
78
|
+
윤한홍의원 등 10인
|
|
79
|
+
<a class="btn_sm" href="#">제안자목록</a>
|
|
80
|
+
</td>
|
|
81
|
+
<td>2026-04-01</td>
|
|
82
|
+
<td class="td_block">
|
|
83
|
+
기획재정위원회
|
|
84
|
+
<div class="m_subject">
|
|
85
|
+
<ul class="m_date">
|
|
86
|
+
<li>입법예고기간 : 2026-04-02~2026-04-11</li>
|
|
87
|
+
</ul>
|
|
88
|
+
</div>
|
|
89
|
+
</td>
|
|
90
|
+
<td>2026-04-02</td>
|
|
91
|
+
<td>2026-04-02~2026-04-11</td>
|
|
92
|
+
<td><a href="#">미리보기</a></td>
|
|
93
|
+
<td>제22대(2024~2028) 제433회</td>
|
|
94
|
+
</tr>
|
|
95
|
+
</tbody>
|
|
96
|
+
</table>
|
|
97
|
+
</div>
|
|
98
|
+
<div class="card-wrap">
|
|
99
|
+
<div class="item">
|
|
100
|
+
<h4>제안이유 및 주요내용</h4>
|
|
101
|
+
<div class="desc">
|
|
102
|
+
제안이유 및 주요내용
|
|
103
|
+
<br/>첫 번째 문장
|
|
104
|
+
<br/>두 번째 문장
|
|
105
|
+
</div>
|
|
106
|
+
</div>
|
|
107
|
+
<div class="item">
|
|
108
|
+
<h4>의견제출 방법</h4>
|
|
109
|
+
<div class="desc">국회 재정경제기획위원회</div>
|
|
110
|
+
</div>
|
|
111
|
+
<div class="item item-means">
|
|
112
|
+
<div class="desc">제출방법: 상단 의견등록 탭 사용</div>
|
|
113
|
+
</div>
|
|
114
|
+
</div>
|
|
115
|
+
`;
|
|
116
|
+
const SEARCH_RESULT_HTML = `
|
|
117
|
+
<div class="board_count">
|
|
118
|
+
<span>전체</span>
|
|
119
|
+
<strong>143</strong>
|
|
120
|
+
<span>건 (2/15 페이지)</span>
|
|
121
|
+
</div>
|
|
122
|
+
<table>
|
|
123
|
+
<tbody>
|
|
124
|
+
<tr>
|
|
125
|
+
<td>42</td>
|
|
126
|
+
<td class="td_block">
|
|
127
|
+
<a href="/napal/lgsltpa/lgsltpaOngoing/view.do?lgsltPaId=PRC_TEST000" class="board_subject">테스트 법률안 (홍길동의원 등 5인)</a>
|
|
128
|
+
</td>
|
|
129
|
+
<td>의원</td>
|
|
130
|
+
<td>법제사법위원회</td>
|
|
131
|
+
<td></td>
|
|
132
|
+
<td>
|
|
133
|
+
<a href="#">버튼</a>
|
|
134
|
+
<a href="https://example.com/file.hwp">hwp</a>
|
|
135
|
+
<a href="https://example.com/file.pdf">pdf</a>
|
|
136
|
+
</td>
|
|
137
|
+
<td></td>
|
|
138
|
+
<td>1,234</td>
|
|
139
|
+
<td></td>
|
|
140
|
+
</tr>
|
|
141
|
+
</tbody>
|
|
142
|
+
</table>
|
|
143
|
+
`;
|
|
144
|
+
// ─── PalParser ──────────────────────────────────────────────────────────────────────────────
|
|
145
|
+
describe('PalParser', () => {
|
|
146
|
+
const parser = new parser_1.PalParser();
|
|
147
|
+
describe('parseTable', () => {
|
|
148
|
+
test('returns [] for invalid html', () => {
|
|
149
|
+
expect(parser.parseTable('asdf')).toHaveLength(0);
|
|
150
|
+
});
|
|
151
|
+
test('returns [] for empty string', () => {
|
|
152
|
+
expect(parser.parseTable('')).toHaveLength(0);
|
|
153
|
+
});
|
|
154
|
+
test('parses all fields from a valid table row', () => {
|
|
155
|
+
const rows = parser.parseTable(TABLE_ROW_HTML);
|
|
156
|
+
expect(rows).toHaveLength(1);
|
|
157
|
+
const row = rows[0];
|
|
158
|
+
expect(row.num).toBe(42);
|
|
159
|
+
expect(row.subject).toBe('테스트 법률안 (홍길동의원 등 5인)');
|
|
160
|
+
expect(row.proposerCategory).toBe('의원');
|
|
161
|
+
expect(row.committee).toBe('법제사법위원회');
|
|
162
|
+
expect(row.numComments).toBe(1234);
|
|
163
|
+
expect(row.link).toContain('PRC_TEST000');
|
|
164
|
+
expect(row.contentId).toBe('PRC_TEST000');
|
|
165
|
+
expect(row.attachments.hwpFile).toBe('https://example.com/file.hwp');
|
|
166
|
+
expect(row.attachments.pdfFile).toBe('https://example.com/file.pdf');
|
|
167
|
+
});
|
|
168
|
+
test('sets link to empty string and contentId to null when anchor has no href', () => {
|
|
169
|
+
const rows = parser.parseTable(TABLE_ROW_NO_LINK_HTML);
|
|
170
|
+
expect(rows).toHaveLength(1);
|
|
171
|
+
expect(rows[0].link).toBe('');
|
|
172
|
+
expect(rows[0].contentId).toBeNull();
|
|
173
|
+
});
|
|
174
|
+
test('parses numComments with comma formatting as a plain number', () => {
|
|
175
|
+
const rows = parser.parseTable(TABLE_ROW_HTML);
|
|
176
|
+
expect(rows[0].numComments).toBe(1234);
|
|
177
|
+
});
|
|
51
178
|
});
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
<div class="
|
|
100
|
-
|
|
101
|
-
<
|
|
102
|
-
<br/>두 번째 문장
|
|
179
|
+
describe('parseContent', () => {
|
|
180
|
+
test('parses all fields from a full fixture', () => {
|
|
181
|
+
const content = parser.parseContent(CONTENT_HTML);
|
|
182
|
+
expect(content.title).toContain('조세특례제한법 일부개정법률안');
|
|
183
|
+
expect(content.proposalReason).toBe('첫 번째 문장\n두 번째 문장');
|
|
184
|
+
expect(content.billNumber).toBe('2218288');
|
|
185
|
+
expect(content.proposer).toBe('윤한홍의원 등 10인');
|
|
186
|
+
expect(content.proposalDate).toBe('2026-04-01');
|
|
187
|
+
expect(content.committee).toBe('기획재정위원회');
|
|
188
|
+
expect(content.referralDate).toBe('2026-04-02');
|
|
189
|
+
expect(content.noticePeriod).toBe('2026-04-02~2026-04-11');
|
|
190
|
+
expect(content.proposalSession).toBe('제22대(2024~2028) 제433회');
|
|
191
|
+
});
|
|
192
|
+
test('strips the repeated heading line from proposal reason', () => {
|
|
193
|
+
const content = parser.parseContent(CONTENT_HTML);
|
|
194
|
+
// The desc starts with "제안이유 및 주요내용" which matches the heading — it must be stripped.
|
|
195
|
+
expect(content.proposalReason).not.toMatch(/제안이유/);
|
|
196
|
+
});
|
|
197
|
+
test('returns empty title and all nulls for empty html', () => {
|
|
198
|
+
const content = parser.parseContent('');
|
|
199
|
+
expect(content.title).toBe('');
|
|
200
|
+
expect(content.proposalReason).toBeNull();
|
|
201
|
+
expect(content.billNumber).toBeNull();
|
|
202
|
+
expect(content.proposer).toBeNull();
|
|
203
|
+
expect(content.proposalDate).toBeNull();
|
|
204
|
+
expect(content.committee).toBeNull();
|
|
205
|
+
expect(content.referralDate).toBeNull();
|
|
206
|
+
expect(content.noticePeriod).toBeNull();
|
|
207
|
+
expect(content.proposalSession).toBeNull();
|
|
208
|
+
});
|
|
209
|
+
test('falls back to h3 when .legislation-heading h3 is absent', () => {
|
|
210
|
+
const content = parser.parseContent('<h3>대체 제목</h3>');
|
|
211
|
+
expect(content.title).toBe('대체 제목');
|
|
212
|
+
});
|
|
213
|
+
test('returns null for all bill info fields when .board-added table is absent', () => {
|
|
214
|
+
const content = parser.parseContent('<div class="legislation-heading"><h3>제목</h3></div>');
|
|
215
|
+
expect(content.billNumber).toBeNull();
|
|
216
|
+
expect(content.proposer).toBeNull();
|
|
217
|
+
expect(content.proposalDate).toBeNull();
|
|
218
|
+
expect(content.committee).toBeNull();
|
|
219
|
+
expect(content.referralDate).toBeNull();
|
|
220
|
+
expect(content.noticePeriod).toBeNull();
|
|
221
|
+
expect(content.proposalSession).toBeNull();
|
|
222
|
+
});
|
|
223
|
+
test('returns null proposalReason when desc is empty', () => {
|
|
224
|
+
const content = parser.parseContent(`
|
|
225
|
+
<div class="card-wrap">
|
|
226
|
+
<div class="item">
|
|
227
|
+
<h4>제안이유 및 주요내용</h4>
|
|
228
|
+
<div class="desc"></div>
|
|
103
229
|
</div>
|
|
104
230
|
</div>
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
231
|
+
`);
|
|
232
|
+
expect(content.proposalReason).toBeNull();
|
|
233
|
+
});
|
|
234
|
+
test('keeps proposal reason as-is when first line does not repeat the heading', () => {
|
|
235
|
+
const content = parser.parseContent(`
|
|
236
|
+
<div class="card-wrap">
|
|
237
|
+
<div class="item">
|
|
238
|
+
<h4>제안이유 및 주요내용</h4>
|
|
239
|
+
<div class="desc">
|
|
240
|
+
본문 첫 줄
|
|
241
|
+
<br/>본문 두 번째 줄
|
|
242
|
+
</div>
|
|
243
|
+
</div>
|
|
111
244
|
</div>
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
245
|
+
`);
|
|
246
|
+
expect(content.proposalReason).toBe('본문 첫 줄\n본문 두 번째 줄');
|
|
247
|
+
});
|
|
248
|
+
test('ignores items without an h4 heading', () => {
|
|
249
|
+
expect(() => parser.parseContent(`
|
|
250
|
+
<div class="card-wrap">
|
|
251
|
+
<div class="item">
|
|
252
|
+
<div class="desc">heading 없는 항목</div>
|
|
253
|
+
</div>
|
|
254
|
+
</div>
|
|
255
|
+
`)).not.toThrow();
|
|
256
|
+
});
|
|
124
257
|
});
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
258
|
+
describe('parseSearchResult', () => {
|
|
259
|
+
test('parses total, totalPages, currentPage, and items from fixture', () => {
|
|
260
|
+
const result = parser.parseSearchResult(SEARCH_RESULT_HTML);
|
|
261
|
+
expect(result.total).toBe(143);
|
|
262
|
+
expect(result.totalPages).toBe(15);
|
|
263
|
+
expect(result.currentPage).toBe(2);
|
|
264
|
+
expect(result.items).toHaveLength(1);
|
|
265
|
+
expect(result.items[0].subject).toBe('테스트 법률안 (홍길동의원 등 5인)');
|
|
266
|
+
});
|
|
267
|
+
test('returns zero total and empty items for empty html', () => {
|
|
268
|
+
const result = parser.parseSearchResult('');
|
|
269
|
+
expect(result.total).toBe(0);
|
|
270
|
+
expect(result.totalPages).toBe(0);
|
|
271
|
+
expect(result.currentPage).toBe(1);
|
|
272
|
+
expect(result.items).toHaveLength(0);
|
|
273
|
+
});
|
|
274
|
+
test('falls back to items.length when count structure is absent', () => {
|
|
275
|
+
const result = parser.parseSearchResult(TABLE_ROW_HTML);
|
|
276
|
+
expect(result.total).toBe(1);
|
|
277
|
+
expect(result.totalPages).toBe(1);
|
|
278
|
+
expect(result.currentPage).toBe(1);
|
|
279
|
+
});
|
|
128
280
|
});
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
281
|
+
});
|
|
282
|
+
// ─── PalCrawl ─────────────────────────────────────────────────────────────────
|
|
283
|
+
describe('PalCrawl', () => {
|
|
284
|
+
const palCrawl = new pal_1.PalCrawl();
|
|
285
|
+
// ── constructor ──────────────────────────────────────────────────────────────
|
|
286
|
+
describe('constructor', () => {
|
|
287
|
+
test('creates instance without config', () => {
|
|
288
|
+
expect(new pal_1.PalCrawl()).toBeInstanceOf(pal_1.PalCrawl);
|
|
289
|
+
});
|
|
290
|
+
test('creates instance with empty config object', () => {
|
|
291
|
+
expect(new pal_1.PalCrawl({})).toBeInstanceOf(pal_1.PalCrawl);
|
|
292
|
+
});
|
|
293
|
+
test('accepts custom userAgent', () => {
|
|
294
|
+
expect(new pal_1.PalCrawl({ userAgent: 'Custom Agent' })).toBeInstanceOf(pal_1.PalCrawl);
|
|
295
|
+
});
|
|
296
|
+
test('accepts custom timeout', () => {
|
|
297
|
+
expect(new pal_1.PalCrawl({ timeout: 5000 })).toBeInstanceOf(pal_1.PalCrawl);
|
|
298
|
+
});
|
|
299
|
+
test('accepts custom retryCount', () => {
|
|
300
|
+
expect(new pal_1.PalCrawl({ retryCount: 5 })).toBeInstanceOf(pal_1.PalCrawl);
|
|
301
|
+
});
|
|
302
|
+
test('accepts custom headers', () => {
|
|
303
|
+
expect(new pal_1.PalCrawl({ customHeaders: { 'Accept-Language': 'ko-KR' } })).toBeInstanceOf(pal_1.PalCrawl);
|
|
304
|
+
});
|
|
305
|
+
test('accepts all config options together', () => {
|
|
306
|
+
const config = {
|
|
307
|
+
userAgent: 'Custom Agent',
|
|
308
|
+
timeout: 15000,
|
|
309
|
+
retryCount: 2,
|
|
310
|
+
customHeaders: { Accept: 'application/json' },
|
|
311
|
+
};
|
|
312
|
+
expect(new pal_1.PalCrawl(config)).toBeInstanceOf(pal_1.PalCrawl);
|
|
313
|
+
});
|
|
143
314
|
});
|
|
144
|
-
|
|
145
|
-
|
|
315
|
+
// ── parseTable / parseContent (delegation) ───────────────────────────────────
|
|
316
|
+
describe('parseTable', () => {
|
|
317
|
+
test('returns [] for invalid html', () => {
|
|
318
|
+
expect(palCrawl.parseTable('asdf')).toHaveLength(0);
|
|
319
|
+
});
|
|
320
|
+
test('delegates to PalParser and returns parsed rows', () => {
|
|
321
|
+
const rows = palCrawl.parseTable(TABLE_ROW_HTML);
|
|
322
|
+
expect(rows).toHaveLength(1);
|
|
323
|
+
expect(rows[0].num).toBe(42);
|
|
324
|
+
expect(rows[0].subject).toBe('테스트 법률안 (홍길동의원 등 5인)');
|
|
325
|
+
expect(rows[0].contentId).toBe('PRC_TEST000');
|
|
326
|
+
});
|
|
146
327
|
});
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
expect(
|
|
151
|
-
expect(
|
|
152
|
-
expect(
|
|
153
|
-
expect(e.committee).not.toContain('�');
|
|
154
|
-
expect(e.contentId).not.toBeNull();
|
|
155
|
-
expect(e.contentId?.startsWith('PRC_')).toBe(true);
|
|
328
|
+
describe('parseContent', () => {
|
|
329
|
+
test('delegates to PalParser and returns parsed content', () => {
|
|
330
|
+
const content = palCrawl.parseContent(CONTENT_HTML);
|
|
331
|
+
expect(content.title).toContain('조세특례제한법 일부개정법률안');
|
|
332
|
+
expect(content.billNumber).toBe('2218288');
|
|
333
|
+
expect(content.proposer).toBe('윤한홍의원 등 10인');
|
|
156
334
|
});
|
|
157
335
|
});
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
336
|
+
// ── ongoing legislative notices (integration) ────────────────────────────────
|
|
337
|
+
describe('ongoing legislative notices', () => {
|
|
338
|
+
let listHtml = '';
|
|
339
|
+
let ongoingTable = [];
|
|
340
|
+
let contentHtml = '';
|
|
341
|
+
let fixedContent;
|
|
342
|
+
beforeAll(async () => {
|
|
343
|
+
[listHtml, contentHtml] = await Promise.all([
|
|
344
|
+
palCrawl.getPalHTML(),
|
|
345
|
+
palCrawl.getContentHTML(FIXED_ONGOING_CONTENT_ID),
|
|
346
|
+
]);
|
|
347
|
+
ongoingTable = palCrawl.parseTable(listHtml);
|
|
348
|
+
fixedContent = palCrawl.parseContent(contentHtml);
|
|
349
|
+
debugLog('ongoingTable', ongoingTable);
|
|
350
|
+
debugLog('fixedContent', fixedContent);
|
|
351
|
+
});
|
|
352
|
+
test('getPalHTML: returns a non-garbled html string', () => {
|
|
353
|
+
expect(typeof listHtml).toBe('string');
|
|
354
|
+
expect(listHtml.length).toBeGreaterThan(0);
|
|
355
|
+
expect(listHtml).not.toContain('�');
|
|
356
|
+
});
|
|
357
|
+
test('get: returns 10 items per page', async () => {
|
|
358
|
+
const result = await palCrawl.get();
|
|
359
|
+
expect(result).toHaveLength(10);
|
|
360
|
+
});
|
|
361
|
+
test('get: each item has valid structure and no garbled text', () => {
|
|
362
|
+
ongoingTable.forEach((e) => {
|
|
363
|
+
expect(e.subject).not.toContain('�');
|
|
364
|
+
expect(e.proposerCategory).not.toContain('�');
|
|
365
|
+
expect(e.committee).not.toContain('�');
|
|
366
|
+
expect(e.attachments.pdfFile).not.toBe('');
|
|
367
|
+
expect(e.attachments.hwpFile).not.toBe('');
|
|
368
|
+
expect(e.contentId).not.toBeNull();
|
|
369
|
+
expect(e.contentId?.startsWith('PRC_')).toBe(true);
|
|
370
|
+
});
|
|
371
|
+
});
|
|
372
|
+
test('getContentHTML: returns non-empty html for known id', () => {
|
|
373
|
+
expect(typeof contentHtml).toBe('string');
|
|
374
|
+
expect(contentHtml.length).toBeGreaterThan(0);
|
|
375
|
+
expect(contentHtml).not.toContain('�');
|
|
376
|
+
});
|
|
377
|
+
test('getContent: parses all fields from known id without garbled text', () => {
|
|
378
|
+
expect(fixedContent.title.length).toBeGreaterThan(0);
|
|
379
|
+
expect(fixedContent.title).not.toContain('�');
|
|
380
|
+
expect(fixedContent.proposalReason).not.toBeNull();
|
|
381
|
+
expect(fixedContent.proposalReason?.length ?? 0).toBeGreaterThan(0);
|
|
382
|
+
expect(fixedContent.proposalReason ?? '').not.toContain('�');
|
|
383
|
+
expect(fixedContent.billNumber).not.toBeNull();
|
|
384
|
+
expect(fixedContent.proposer).not.toBeNull();
|
|
385
|
+
expect(fixedContent.proposalDate).not.toBeNull();
|
|
386
|
+
expect(fixedContent.committee).not.toBeNull();
|
|
387
|
+
expect(fixedContent.referralDate).not.toBeNull();
|
|
388
|
+
expect(fixedContent.noticePeriod).not.toBeNull();
|
|
389
|
+
expect(fixedContent.proposalSession).not.toBeNull();
|
|
390
|
+
});
|
|
391
|
+
test('getContentHTML: throws when id is an empty string', async () => {
|
|
392
|
+
await expect(palCrawl.getContentHTML('')).rejects.toThrow('id is required');
|
|
393
|
+
});
|
|
394
|
+
test('getContentHTML: throws when id is whitespace only', async () => {
|
|
395
|
+
await expect(palCrawl.getContentHTML(' ')).rejects.toThrow('id is required');
|
|
396
|
+
});
|
|
164
397
|
});
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
398
|
+
// ── done legislative notices (integration) ───────────────────────────────────
|
|
399
|
+
describe('done legislative notices', () => {
|
|
400
|
+
let doneHtml = '';
|
|
401
|
+
let doneTable = [];
|
|
402
|
+
let doneContentHtml = '';
|
|
403
|
+
let doneContent;
|
|
404
|
+
beforeAll(async () => {
|
|
405
|
+
[doneHtml, doneContentHtml] = await Promise.all([
|
|
406
|
+
palCrawl.getDoneHTML(),
|
|
407
|
+
palCrawl.getDoneContentHTML(FIXED_DONE_CONTENT_ID),
|
|
408
|
+
]);
|
|
409
|
+
doneTable = palCrawl.parseTable(doneHtml);
|
|
410
|
+
doneContent = palCrawl.parseContent(doneContentHtml);
|
|
411
|
+
debugLog('doneTable', doneTable);
|
|
412
|
+
debugLog('doneContent', doneContent);
|
|
413
|
+
});
|
|
414
|
+
test('getDoneHTML: returns a non-garbled html string', () => {
|
|
415
|
+
expect(typeof doneHtml).toBe('string');
|
|
416
|
+
expect(doneHtml.length).toBeGreaterThan(0);
|
|
417
|
+
expect(doneHtml).not.toContain('�');
|
|
418
|
+
});
|
|
419
|
+
test('getDone: returns 10 items per page', async () => {
|
|
420
|
+
const result = await palCrawl.getDone();
|
|
421
|
+
expect(result).toHaveLength(10);
|
|
422
|
+
});
|
|
423
|
+
test('getDone: each item has valid structure and no garbled text', () => {
|
|
424
|
+
doneTable.forEach((e) => {
|
|
425
|
+
expect(e.subject).not.toContain('�');
|
|
426
|
+
expect(e.proposerCategory).not.toContain('�');
|
|
427
|
+
expect(e.committee).not.toContain('�');
|
|
428
|
+
expect(e.contentId).not.toBeNull();
|
|
429
|
+
expect(e.contentId?.startsWith('PRC_')).toBe(true);
|
|
430
|
+
});
|
|
431
|
+
});
|
|
432
|
+
test('getDoneContentHTML: returns non-empty html for known id', () => {
|
|
433
|
+
expect(typeof doneContentHtml).toBe('string');
|
|
434
|
+
expect(doneContentHtml.length).toBeGreaterThan(0);
|
|
435
|
+
expect(doneContentHtml).not.toContain('�');
|
|
436
|
+
});
|
|
437
|
+
test('getDoneContent: parses title and bill info from known id', () => {
|
|
438
|
+
expect(doneContent.title.length).toBeGreaterThan(0);
|
|
439
|
+
expect(doneContent.title).not.toContain('�');
|
|
440
|
+
expect(doneContent.billNumber).not.toBeNull();
|
|
441
|
+
expect(doneContent.proposer).not.toBeNull();
|
|
442
|
+
expect(doneContent.proposalDate).not.toBeNull();
|
|
443
|
+
expect(doneContent.committee).not.toBeNull();
|
|
444
|
+
});
|
|
445
|
+
test('getDoneContentHTML: throws when id is an empty string', async () => {
|
|
446
|
+
await expect(palCrawl.getDoneContentHTML('')).rejects.toThrow('id is required');
|
|
447
|
+
});
|
|
448
|
+
test('getDoneContentHTML: throws when id is whitespace only', async () => {
|
|
449
|
+
await expect(palCrawl.getDoneContentHTML(' ')).rejects.toThrow('id is required');
|
|
450
|
+
});
|
|
171
451
|
});
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
452
|
+
// ── search / searchDone (integration) ─────────────────────────────────────────
|
|
453
|
+
describe('search / searchDone', () => {
|
|
454
|
+
test('search: returns ISearchResult with total, totalPages, currentPage', async () => {
|
|
455
|
+
const result = await palCrawl.search();
|
|
456
|
+
expect(result.total).toBeGreaterThan(0);
|
|
457
|
+
expect(result.totalPages).toBeGreaterThan(0);
|
|
458
|
+
expect(result.currentPage).toBe(1);
|
|
459
|
+
expect(result.items.length).toBeGreaterThan(0);
|
|
460
|
+
});
|
|
461
|
+
test('search: billName filter returns matching items', async () => {
|
|
462
|
+
const result = await palCrawl.search({ billName: '도로교통' });
|
|
463
|
+
expect(result.items.length).toBeGreaterThan(0);
|
|
464
|
+
result.items.forEach((item) => {
|
|
465
|
+
expect(item.subject).toMatch(/도로교통/);
|
|
466
|
+
});
|
|
467
|
+
});
|
|
468
|
+
test('search: sortCol=BILL_NAME sortGbn=ASC returns alphabetically sorted items', async () => {
|
|
469
|
+
const result = await palCrawl.search({
|
|
470
|
+
sortCol: 'BILL_NAME',
|
|
471
|
+
sortGbn: 'ASC',
|
|
472
|
+
pageUnit: 5,
|
|
473
|
+
});
|
|
474
|
+
expect(result.items.length).toBeGreaterThan(0);
|
|
475
|
+
// First subject should start alphabetically early (가ㅠ)
|
|
476
|
+
expect(result.items[0].subject).not.toContain('오류');
|
|
477
|
+
});
|
|
478
|
+
test('searchDone: returns ISearchResult with large total', async () => {
|
|
479
|
+
const result = await palCrawl.searchDone();
|
|
480
|
+
expect(result.total).toBeGreaterThan(1000);
|
|
481
|
+
expect(result.totalPages).toBeGreaterThan(100);
|
|
482
|
+
expect(result.currentPage).toBe(1);
|
|
483
|
+
});
|
|
484
|
+
test('searchDone: fromAge/toAge filter narrows results', async () => {
|
|
485
|
+
const result = await palCrawl.searchDone({
|
|
486
|
+
fromAge: 22,
|
|
487
|
+
toAge: 22,
|
|
488
|
+
pageUnit: 5,
|
|
489
|
+
});
|
|
490
|
+
// The site always reports the global total in the count area,
|
|
491
|
+
// but the returned items are filtered to the specified session range.
|
|
492
|
+
expect(result.items.length).toBeGreaterThan(0);
|
|
493
|
+
expect(result.items.length).toBeLessThanOrEqual(5);
|
|
494
|
+
});
|
|
181
495
|
});
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
496
|
+
// ── getPage / getDonePage (integration) ─────────────────────────────────────
|
|
497
|
+
describe('getPage / getDonePage', () => {
|
|
498
|
+
test('getPage(1): returns same items as get()', async () => {
|
|
499
|
+
const [page1, legacy] = await Promise.all([
|
|
500
|
+
palCrawl.getPage(1),
|
|
501
|
+
palCrawl.get(),
|
|
502
|
+
]);
|
|
503
|
+
expect(page1).toHaveLength(legacy.length);
|
|
504
|
+
expect(page1[0].subject).toBe(legacy[0].subject);
|
|
505
|
+
});
|
|
506
|
+
test('getPage(2): returns different items from page 1', async () => {
|
|
507
|
+
const [page1, page2] = await Promise.all([
|
|
508
|
+
palCrawl.getPage(1),
|
|
509
|
+
palCrawl.getPage(2),
|
|
510
|
+
]);
|
|
511
|
+
expect(page1.length).toBeGreaterThan(0);
|
|
512
|
+
expect(page2.length).toBeGreaterThan(0);
|
|
513
|
+
expect(page1[0].subject).not.toBe(page2[0].subject);
|
|
514
|
+
});
|
|
515
|
+
test('getPage: pageUnit=20 returns up to 20 items', async () => {
|
|
516
|
+
const items = await palCrawl.getPage(1, 20);
|
|
517
|
+
expect(items.length).toBeLessThanOrEqual(20);
|
|
518
|
+
expect(items.length).toBeGreaterThan(10);
|
|
519
|
+
});
|
|
520
|
+
test('getDonePage(2): returns items different from page 1', async () => {
|
|
521
|
+
const [page1, page2] = await Promise.all([
|
|
522
|
+
palCrawl.getDonePage(1),
|
|
523
|
+
palCrawl.getDonePage(2),
|
|
524
|
+
]);
|
|
525
|
+
expect(page1[0].subject).not.toBe(page2[0].subject);
|
|
526
|
+
});
|
|
527
|
+
});
|
|
528
|
+
// ── getAllPages / getAllDonePages (integration) ──────────────────────────────
|
|
529
|
+
describe('getAllPages / getAllDonePages', () => {
|
|
530
|
+
test('getAllPages: yields exactly maxPages results', async () => {
|
|
531
|
+
const pages = [];
|
|
532
|
+
for await (const page of palCrawl.getAllPages({ pageUnit: 5 }, { maxPages: 2, delayMs: 0 })) {
|
|
533
|
+
pages.push(page);
|
|
534
|
+
}
|
|
535
|
+
expect(pages).toHaveLength(2);
|
|
536
|
+
expect(pages[0].items.length).toBeGreaterThan(0);
|
|
537
|
+
expect(pages[1].items.length).toBeGreaterThan(0);
|
|
538
|
+
});
|
|
539
|
+
test('getAllPages: currentPage increments correctly', async () => {
|
|
540
|
+
const pages = [];
|
|
541
|
+
for await (const page of palCrawl.getAllPages({ pageUnit: 5 }, { maxPages: 3, delayMs: 0 })) {
|
|
542
|
+
pages.push(page);
|
|
543
|
+
}
|
|
544
|
+
expect(pages[0].currentPage).toBe(1);
|
|
545
|
+
expect(pages[1].currentPage).toBe(2);
|
|
546
|
+
expect(pages[2].currentPage).toBe(3);
|
|
547
|
+
});
|
|
548
|
+
test('getAllPages: total and totalPages are consistent across pages', async () => {
|
|
549
|
+
const pages = [];
|
|
550
|
+
for await (const page of palCrawl.getAllPages({}, { maxPages: 2, delayMs: 0 })) {
|
|
551
|
+
pages.push(page);
|
|
552
|
+
}
|
|
553
|
+
expect(pages[0].total).toBe(pages[1].total);
|
|
554
|
+
expect(pages[0].totalPages).toBe(pages[1].totalPages);
|
|
555
|
+
});
|
|
556
|
+
test('getAllPages: page 2 items differ from page 1', async () => {
|
|
557
|
+
const pages = [];
|
|
558
|
+
for await (const page of palCrawl.getAllPages({ pageUnit: 5 }, { maxPages: 2, delayMs: 0 })) {
|
|
559
|
+
pages.push(page);
|
|
560
|
+
}
|
|
561
|
+
expect(pages[0].items[0].subject).not.toBe(pages[1].items[0].subject);
|
|
562
|
+
});
|
|
563
|
+
test('getAllPages: concurrency > 1 yields pages in order', async () => {
|
|
564
|
+
const pages = [];
|
|
565
|
+
for await (const page of palCrawl.getAllPages({ pageUnit: 5 }, { maxPages: 4, delayMs: 0, concurrency: 2 })) {
|
|
566
|
+
pages.push(page);
|
|
567
|
+
}
|
|
568
|
+
expect(pages).toHaveLength(4);
|
|
569
|
+
for (let i = 0; i < pages.length; i++) {
|
|
570
|
+
expect(pages[i].currentPage).toBe(i + 1);
|
|
571
|
+
}
|
|
572
|
+
});
|
|
573
|
+
test('getAllDonePages: yields 2 pages of done notices', async () => {
|
|
574
|
+
const pages = [];
|
|
575
|
+
for await (const page of palCrawl.getAllDonePages({ pageUnit: 5 }, { maxPages: 2, delayMs: 0 })) {
|
|
576
|
+
pages.push(page);
|
|
577
|
+
}
|
|
578
|
+
expect(pages).toHaveLength(2);
|
|
579
|
+
expect(pages[0].total).toBeGreaterThan(1000);
|
|
580
|
+
expect(pages[0].items[0].subject).not.toBe(pages[1].items[0].subject);
|
|
581
|
+
});
|
|
193
582
|
});
|
|
194
583
|
});
|
|
195
584
|
//# sourceMappingURL=pal.test.js.map
|