x2ch 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/x2ch.rb +183 -0
- metadata +45 -0
data/lib/x2ch.rb
ADDED
@@ -0,0 +1,183 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'open-uri'
|
4
|
+
require 'kconv'
|
5
|
+
|
6
|
+
module X2CH
|
7
|
+
class Bbs
|
8
|
+
attr_accessor :categories
|
9
|
+
|
10
|
+
def initialize()
|
11
|
+
@categories = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def [](cname)
|
15
|
+
@categories.each{|c|
|
16
|
+
return c if c.name == cname
|
17
|
+
}
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
|
21
|
+
def push(category)
|
22
|
+
@categories << category
|
23
|
+
end
|
24
|
+
|
25
|
+
def each(&blk)
|
26
|
+
@categories.each{|c|
|
27
|
+
yield c
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.load()
|
32
|
+
BbsMenu.parse(BbsMenu.download)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
class Category
|
37
|
+
attr_accessor :name, :boards
|
38
|
+
|
39
|
+
def initialize(name)
|
40
|
+
@name = name
|
41
|
+
@boards = []
|
42
|
+
end
|
43
|
+
|
44
|
+
def [](bname)
|
45
|
+
@boards.each{|b|
|
46
|
+
return b if b.name == bname
|
47
|
+
}
|
48
|
+
nil
|
49
|
+
end
|
50
|
+
|
51
|
+
def push(board)
|
52
|
+
@boards << board
|
53
|
+
end
|
54
|
+
|
55
|
+
def each(&blk)
|
56
|
+
@boards.each{|b|
|
57
|
+
yield b
|
58
|
+
}
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
class Board
|
63
|
+
attr_accessor :url, :name
|
64
|
+
|
65
|
+
def initialize(url, name)
|
66
|
+
@url, @name = url, name
|
67
|
+
end
|
68
|
+
|
69
|
+
def threads()
|
70
|
+
Subject.parse(@url, Subject.download(@url + '/subject.txt'))
|
71
|
+
end
|
72
|
+
|
73
|
+
def each(&blk)
|
74
|
+
threads.each{|t|
|
75
|
+
yield t
|
76
|
+
}
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
class Thread
|
81
|
+
attr_accessor :url, :name, :num
|
82
|
+
|
83
|
+
def initialize(url, dat, name, num)
|
84
|
+
@url, @dat, @name, @num = url, dat, name, num
|
85
|
+
end
|
86
|
+
|
87
|
+
def posts
|
88
|
+
Dat.parse(Dat.download(@url + "dat/" + @dat))
|
89
|
+
end
|
90
|
+
|
91
|
+
def each(&blk)
|
92
|
+
posts.each{|p|
|
93
|
+
yield p
|
94
|
+
}
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
class Post
|
99
|
+
attr_accessor :name, :mail, :metadata, :body
|
100
|
+
|
101
|
+
def initialize(name, mail, metadata, body)
|
102
|
+
@name, @mail, @metadata, @body = name, mail, metadata, body
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
class Agent
|
107
|
+
def self.download(url)
|
108
|
+
open(url){|f| f.read}.toutf8
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
class BbsMenu
|
113
|
+
IGNORE_CATEGORIES = ['特別企画', 'チャット', 'ツール類']
|
114
|
+
IGNORE_BOARDS = ['2chプロジェクト', 'いろいろランク']
|
115
|
+
|
116
|
+
def self.download
|
117
|
+
Agent.download("http://menu.2ch.net/bbsmenu.html")
|
118
|
+
end
|
119
|
+
|
120
|
+
def self.parse(html)
|
121
|
+
bbs = Bbs.new
|
122
|
+
category = nil
|
123
|
+
html.each_line{|l|
|
124
|
+
cname = l.match(/<BR><BR><B>(.+?)<\/B><BR>/).to_a[1]
|
125
|
+
if cname
|
126
|
+
if IGNORE_CATEGORIES.include?(cname)
|
127
|
+
category = nil
|
128
|
+
else
|
129
|
+
category = Category.new(cname)
|
130
|
+
bbs.push(category)
|
131
|
+
end
|
132
|
+
|
133
|
+
next
|
134
|
+
end
|
135
|
+
|
136
|
+
next unless category
|
137
|
+
|
138
|
+
b = l.match(/<A HREF=(http:\/\/.*(?:\.2ch\.net|\.bbspink\.com).+\/)>(.+)<\/A>/).to_a
|
139
|
+
if b[0]
|
140
|
+
next if IGNORE_BOARDS.include?(b[2])
|
141
|
+
|
142
|
+
board = Board.new(b[1], b[2])
|
143
|
+
category.push(board)
|
144
|
+
end
|
145
|
+
}
|
146
|
+
bbs
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
class Subject
|
151
|
+
def self.download(url)
|
152
|
+
Agent.download(url)
|
153
|
+
end
|
154
|
+
|
155
|
+
def self.parse(url, subject)
|
156
|
+
threads = []
|
157
|
+
subject.each_line{|l|
|
158
|
+
m = l.match(/^(\d+\.dat)<>(.+)\((\d+)\)$/).to_a
|
159
|
+
if m[0]
|
160
|
+
threads << Thread.new(url, m[1], m[2], m[3].to_i)
|
161
|
+
end
|
162
|
+
}
|
163
|
+
threads
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
class Dat
|
168
|
+
def self.download(url)
|
169
|
+
Agent.download(url)
|
170
|
+
end
|
171
|
+
|
172
|
+
def self.parse(dat)
|
173
|
+
posts = []
|
174
|
+
dat.each_line{|l|
|
175
|
+
m = l.match(/^(.+?)<>(.*?)<>(.*?)<>(.+)<>.*$/).to_a
|
176
|
+
if m[0]
|
177
|
+
posts << Post.new(m[1], m[2], m[3], m[4])
|
178
|
+
end
|
179
|
+
}
|
180
|
+
posts
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
metadata
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: x2ch
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.9.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- xmisao
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-06-21 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description: 2ch downloader and parser library
|
15
|
+
email: mail@xmisao.com
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- lib/x2ch.rb
|
21
|
+
homepage: http://rubygems.org/gems/x2ch
|
22
|
+
licenses: []
|
23
|
+
post_install_message:
|
24
|
+
rdoc_options: []
|
25
|
+
require_paths:
|
26
|
+
- lib
|
27
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
34
|
+
none: false
|
35
|
+
requirements:
|
36
|
+
- - ! '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
requirements: []
|
40
|
+
rubyforge_project:
|
41
|
+
rubygems_version: 1.8.23
|
42
|
+
signing_key:
|
43
|
+
specification_version: 3
|
44
|
+
summary: 2ch downloader and parser library
|
45
|
+
test_files: []
|