fundler 0.2.6 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/fundler +24 -17
- data/lib/fundler.rb +2 -5
- data/lib/fundler/bookmarks_reader.rb +48 -115
- data/lib/fundler/fundler_utils.rb +113 -1
- data/lib/fundler/version.rb +1 -1
- data/spec/fundler_bookmarks_spec.rb +18 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da0902defdfad3f010e5eaccb203cb54c36d71c6
|
4
|
+
data.tar.gz: 8154e8189a025975c2703612b5ab45ecba37b59a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5d9adfc90611ab96b05781356f0467a2d2309224f87ba78ccb9bc5f4ad20f0ca6b0e561c93f73fa6658702e749ecf0fdf7bd9cef56ae68942f6ceebac817aa17
|
7
|
+
data.tar.gz: 638b975cd75a57ad254ca5782afb9e5f2b72f79de16dd3f543f10e127e7c72b0e758279bb8ab554f6030a5fc8f47391d219b301921e3bad71eeab001a565bc15
|
data/bin/fundler
CHANGED
@@ -11,7 +11,7 @@ class FundlerProgram
|
|
11
11
|
attr_reader :input
|
12
12
|
|
13
13
|
def err msg
|
14
|
-
$stderr.puts "*
|
14
|
+
$stderr.puts "* : #{msg}"
|
15
15
|
exit 1
|
16
16
|
end
|
17
17
|
|
@@ -29,51 +29,59 @@ class FundlerProgram
|
|
29
29
|
@debug = false
|
30
30
|
|
31
31
|
# Parser
|
32
|
+
options = {}
|
32
33
|
optparse = OptionParser.new do |opts|
|
33
34
|
opts.banner = "Usage: #{opts.program_name} [OPTIONS] "
|
34
|
-
opts.define_head 'file name cleaner'
|
35
|
+
opts.define_head 'Fundler: file name cleaner and bookmarks utility'
|
36
|
+
opts.program_name = 'Fundler'
|
35
37
|
opts.separator ''
|
36
38
|
|
37
39
|
opts.on('-d', '--debug', "Turn on debugging.") do
|
38
40
|
@debug = true
|
39
41
|
end
|
40
42
|
|
41
|
-
opts.on('-o', '--output FILE', String,
|
42
|
-
|
43
|
-
|
44
|
-
end
|
43
|
+
#opts.on('-o', '--output FILE', String,
|
44
|
+
# "Where to output the merged ruby script.") do |f|
|
45
|
+
# @output = File.open(f, 'w') unless '-' == f
|
46
|
+
#end
|
45
47
|
|
46
48
|
opts.on('-c', '--clean_names',
|
47
49
|
'clean, squeeze, downcase the filenames in pwd') do
|
48
50
|
debug "Cleaning the pwd: #{fundler.pwd}"
|
49
51
|
fundler.clean
|
50
52
|
end
|
53
|
+
|
51
54
|
opts.on('-l', '--list', 'listing files in pwd') do
|
52
55
|
debug "List files in #{fundler.pwd}"
|
53
56
|
fundler.list
|
54
57
|
end
|
55
58
|
|
56
|
-
opts.on('-b', '--bookmarks',
|
59
|
+
opts.on('-b', '--bookmarks FORMAT', String,
|
60
|
+
'drop firefox bookmarks in txt, html, markdown, json') do |format|
|
57
61
|
debug "Bookmarks util"
|
58
|
-
|
62
|
+
options[:format] = format
|
63
|
+
fundler.bookmarks(format)
|
59
64
|
end
|
65
|
+
|
60
66
|
opts.on_tail('-V', '--version', 'Show the version.') do
|
61
|
-
|
62
|
-
exit
|
67
|
+
err "#{opts.program_name} version: #{Fundler::VERSION}"
|
63
68
|
end
|
64
69
|
|
65
70
|
opts.on_tail('-h', '--help', 'This Help.') do
|
66
|
-
|
67
|
-
exit
|
71
|
+
err opts
|
68
72
|
end
|
69
73
|
end
|
70
74
|
|
71
|
-
|
75
|
+
begin
|
76
|
+
arguments = optparse.parse!(ARGV)
|
77
|
+
rescue OptionParser::ParseError
|
78
|
+
err optparse.help
|
79
|
+
end
|
72
80
|
|
73
|
-
if arguments.size == 0
|
74
|
-
#
|
81
|
+
if arguments.size == 0
|
82
|
+
#err "Argument needed!\n#{optparse.banner}"
|
75
83
|
elsif arguments.size > 1
|
76
|
-
err "You can only specify one
|
84
|
+
err "You can only specify one format a time"
|
77
85
|
end
|
78
86
|
|
79
87
|
if '-' == arguments.first
|
@@ -96,7 +104,6 @@ class FundlerProgram
|
|
96
104
|
end
|
97
105
|
|
98
106
|
#if __FILE__ == $0
|
99
|
-
p Time.now
|
100
107
|
FundlerProgram.new.run
|
101
108
|
#end
|
102
109
|
|
data/lib/fundler.rb
CHANGED
@@ -13,13 +13,10 @@ class Fundler
|
|
13
13
|
def pwd
|
14
14
|
get_pwd
|
15
15
|
end
|
16
|
-
def bookmarks
|
17
|
-
drop_bookmarks
|
16
|
+
def bookmarks(format)
|
17
|
+
drop_bookmarks(format)
|
18
18
|
end
|
19
19
|
def list
|
20
20
|
get_list
|
21
21
|
end
|
22
|
-
def drop_bookmarks_file(format = :plain)
|
23
|
-
drop_bookmarks(format)
|
24
|
-
end
|
25
22
|
end
|
@@ -1,105 +1,18 @@
|
|
1
1
|
# coding: utf-8
|
2
|
-
require 'sqlite3'
|
3
|
-
|
4
|
-
=begin rdoc
|
5
|
-
|
6
|
-
== Schema
|
7
|
-
http://people.mozilla.org/~dietrich/places-erd.png
|
8
|
-
|
9
|
-
== References
|
10
|
-
http://stackoverflow.com/questions/464516/firefox-bookmarks-sqlite-structure
|
11
|
-
https://developer.mozilla.org/en-US/docs/Places
|
12
|
-
https://developer.mozilla.org/en-US/docs/Retrieving_part_of_the_bookmarks_tree
|
13
|
-
https://developer.mozilla.org/en-US/docs/XPCOM_Interface_Reference/nsINavBookmarksService
|
14
|
-
http://davidkoepi.wordpress.com/2010/11/27/firefoxforensics/
|
15
|
-
|
16
|
-
== Queries
|
17
|
-
|
18
|
-
select moz_places.url, moz_bookmarks.title
|
19
|
-
from moz_places,moz_bookmarks
|
20
|
-
where moz_places.id = moz_bookmarks.fk and moz_bookmarks.title != '';
|
21
2
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
left join moz_places on (fk = moz_places.id);
|
3
|
+
require 'sqlite3'
|
4
|
+
require 'json'
|
5
|
+
require_relative 'fundler_utils'
|
26
6
|
|
27
|
-
|
28
|
-
from moz_places, moz_historyvisits
|
29
|
-
where moz_historyvisits.place_id = moz_places.id
|
30
|
-
order by moz_historyvisits.visit_date desc;
|
31
|
-
=end
|
7
|
+
#TODO: make private methods... well, private!
|
32
8
|
|
33
9
|
module BookmarksReader
|
34
10
|
|
11
|
+
include QueryConstants
|
12
|
+
|
35
13
|
attr_reader :db
|
36
14
|
|
37
15
|
MOZILLA_FIREFOX_CONF_DIR = File.expand_path('~') + '/.mozilla/firefox/'
|
38
|
-
|
39
|
-
# all bookmarks
|
40
|
-
BOOKMARKS_QUERY = <<-SQL
|
41
|
-
SELECT DISTINCT
|
42
|
-
moz_places.url AS url,
|
43
|
-
moz_bookmarks.title AS title,
|
44
|
-
moz_items_annos.content AS description
|
45
|
-
FROM
|
46
|
-
moz_places,
|
47
|
-
moz_bookmarks,
|
48
|
-
moz_items_annos,
|
49
|
-
moz_anno_attributes
|
50
|
-
WHERE
|
51
|
-
moz_anno_attributes.name = 'bookmarkProperties/description' AND
|
52
|
-
moz_items_annos.anno_attribute_id = moz_anno_attributes.id AND
|
53
|
-
moz_items_annos.item_id = moz_bookmarks.id AND
|
54
|
-
moz_places.id = moz_bookmarks.fk AND
|
55
|
-
moz_places.id IN (
|
56
|
-
SELECT DISTINCT fk
|
57
|
-
FROM moz_bookmarks
|
58
|
-
WHERE parent IN (
|
59
|
-
SELECT moz_bookmarks.id
|
60
|
-
FROM moz_bookmarks, moz_bookmarks_roots
|
61
|
-
WHERE moz_bookmarks_roots.root_name = 'tags'
|
62
|
-
AND moz_bookmarks.parent = moz_bookmarks_roots.folder_id
|
63
|
-
)
|
64
|
-
)
|
65
|
-
ORDER BY UPPER(moz_bookmarks.title) ASC
|
66
|
-
SQL
|
67
|
-
|
68
|
-
# tag.id|tag.title
|
69
|
-
QUERY_FOR_TAGS = <<-SQL
|
70
|
-
SELECT id, title FROM moz_bookmarks WHERE parent = 4;
|
71
|
-
SQL
|
72
|
-
|
73
|
-
# all bookmarks with the "given" tag.id
|
74
|
-
QUERY_BY_TAG = <<-SQL
|
75
|
-
SELECT moz_places.id, moz_places.url, moz_places.title, moz_bookmarks.parent
|
76
|
-
FROM moz_places
|
77
|
-
LEFT OUTER JOIN moz_bookmarks
|
78
|
-
ON moz_places.id = moz_bookmarks.fk
|
79
|
-
WHERE moz_bookmarks.parent = @tag_id;
|
80
|
-
SQL
|
81
|
-
|
82
|
-
# bookmark.id|bookmark.fk|bookmark.title
|
83
|
-
# ...
|
84
|
-
#=> 1933|1387|SQLite Home Page
|
85
|
-
# ...
|
86
|
-
QUERY_FOR_SIMPLE_BOOKMARKS = <<-SQL
|
87
|
-
SELECT id,fk,title FROM moz_bookmarks WHERE parent = 2;
|
88
|
-
SQL
|
89
|
-
|
90
|
-
# select the link between bookmars and tags
|
91
|
-
# fk_id = 1387
|
92
|
-
#=> 1934|1387|1144
|
93
|
-
QUERY_FOR_LINK = <<-SQL
|
94
|
-
SELECT id,fk,parent FROM moz_bookmarks WHERE title is null AND fk = @fk_id;
|
95
|
-
SQL
|
96
|
-
|
97
|
-
# tag.id|tag.title
|
98
|
-
# parent = 1144
|
99
|
-
#=> 1144|coding
|
100
|
-
QUERY_FOR_TAG_BY_ID = <<-SQL
|
101
|
-
SELECT id, title FROM moz_bookmarks WHERE parent = 4 AND id = @parent;
|
102
|
-
SQL
|
103
16
|
|
104
17
|
def find_bookmarks_by_tag(tag_id)
|
105
18
|
query = QUERY_BY_TAG
|
@@ -124,28 +37,49 @@ module BookmarksReader
|
|
124
37
|
db.get_first_row(query)
|
125
38
|
end
|
126
39
|
|
40
|
+
def stored_bookmarks
|
41
|
+
# humm... I don't like this choice
|
42
|
+
@stored_bookmarks ||= get_bookmarks_with_tags
|
43
|
+
end
|
44
|
+
|
45
|
+
def bookmarks_count
|
46
|
+
stored_bookmarks.size
|
47
|
+
end
|
48
|
+
|
127
49
|
def get_bookmarks_with_tags
|
128
|
-
|
129
|
-
|
130
|
-
db
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
50
|
+
all = []
|
51
|
+
@stored_bookmarks = {}
|
52
|
+
if db
|
53
|
+
db.execute(QUERY_FOR_SIMPLE_BOOKMARKS) do |bookmark|
|
54
|
+
all << bookmark
|
55
|
+
end
|
56
|
+
all.each do |bookmark|
|
57
|
+
bookmark_id, bookmark_fk, bookmark_title = bookmark
|
58
|
+
if bookmark_fk
|
59
|
+
link_query = QUERY_FOR_LINK.sub(/@fk_id/, bookmark_fk.to_s)
|
60
|
+
links = db.execute(link_query)
|
61
|
+
links.each do |link|
|
62
|
+
link_id, link_fk, link_parent = link
|
63
|
+
tag_query = QUERY_FOR_TAG_BY_ID.gsub(/@parent/, link_parent.to_s)
|
64
|
+
db.execute(tag_query) do |tag|
|
65
|
+
tag_id, tag_title = tag
|
66
|
+
@stored_bookmarks[bookmark_id] = { tag: tag_title, title: bookmark_title }
|
67
|
+
end
|
144
68
|
end
|
145
69
|
end
|
146
70
|
end
|
147
71
|
end
|
148
|
-
|
72
|
+
@stored_bookmarks.extend(HashBookmarksUtil)
|
73
|
+
end
|
74
|
+
|
75
|
+
def get_bookmarks_with_url_title_descr_tag
|
76
|
+
unless stored_bookmarks.empty?
|
77
|
+
db.execute(BOOKMARKS_QUERY).each do |bookmark|
|
78
|
+
url, title, id, description = bookmark
|
79
|
+
@stored_bookmarks[id].merge!({url: url, description: description})
|
80
|
+
end
|
81
|
+
end
|
82
|
+
@stored_bookmarks
|
149
83
|
end
|
150
84
|
|
151
85
|
# +drop_bookmarks+ browse the home directory of the current user,
|
@@ -154,18 +88,17 @@ module BookmarksReader
|
|
154
88
|
def drop_bookmarks(format = :txt)
|
155
89
|
output = []
|
156
90
|
if db
|
157
|
-
|
158
|
-
|
159
|
-
case format
|
91
|
+
get_bookmarks_with_url_title_descr_tag.each_with_index do |row, index|
|
92
|
+
case format.intern
|
160
93
|
when :txt
|
161
|
-
output << "#{index}: #{row
|
94
|
+
output << "#{index}: #{row}"
|
162
95
|
output << '---'
|
163
96
|
when :html
|
164
97
|
puts "TODO"
|
165
98
|
when :markdown
|
166
99
|
puts "TODO"
|
167
100
|
when :json
|
168
|
-
|
101
|
+
output << row.to_json
|
169
102
|
else
|
170
103
|
# exit 1
|
171
104
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
1
2
|
# ---
|
2
3
|
|
3
4
|
require 'fileutils'
|
@@ -47,4 +48,115 @@ module ExitCodeMatchers
|
|
47
48
|
"expect block to call exit(#{code})"
|
48
49
|
end
|
49
50
|
end
|
50
|
-
end
|
51
|
+
end # ExitCodeMatchers
|
52
|
+
|
53
|
+
=begin rdoc
|
54
|
+
|
55
|
+
== Schema
|
56
|
+
http://people.mozilla.org/~dietrich/places-erd.png
|
57
|
+
|
58
|
+
== References
|
59
|
+
http://stackoverflow.com/questions/464516/firefox-bookmarks-sqlite-structure
|
60
|
+
https://developer.mozilla.org/en-US/docs/Places
|
61
|
+
https://developer.mozilla.org/en-US/docs/Retrieving_part_of_the_bookmarks_tree
|
62
|
+
https://developer.mozilla.org/en-US/docs/XPCOM_Interface_Reference/nsINavBookmarksService
|
63
|
+
http://davidkoepi.wordpress.com/2010/11/27/firefoxforensics/
|
64
|
+
|
65
|
+
== Queries
|
66
|
+
|
67
|
+
select moz_places.url, moz_bookmarks.title
|
68
|
+
from moz_places,moz_bookmarks
|
69
|
+
where moz_places.id = moz_bookmarks.fk and moz_bookmarks.title != '';
|
70
|
+
|
71
|
+
select keyword,url
|
72
|
+
from moz_keywords
|
73
|
+
left join moz_bookmarks on (moz_keywords.id = keyword_id)
|
74
|
+
left join moz_places on (fk = moz_places.id);
|
75
|
+
|
76
|
+
select moz_places.url, datetime((moz_historyvisits.visit_date/1000000), ‘unixepoch’, ‘localtime’), moz_historyvisits.visit_type
|
77
|
+
from moz_places, moz_historyvisits
|
78
|
+
where moz_historyvisits.place_id = moz_places.id
|
79
|
+
order by moz_historyvisits.visit_date desc;
|
80
|
+
=end
|
81
|
+
|
82
|
+
module QueryConstants
|
83
|
+
|
84
|
+
# all bookmarks
|
85
|
+
BOOKMARKS_QUERY = <<-SQL
|
86
|
+
SELECT DISTINCT
|
87
|
+
moz_places.url AS url,
|
88
|
+
moz_bookmarks.title AS title,
|
89
|
+
moz_bookmarks.id AS id,
|
90
|
+
moz_items_annos.content AS description
|
91
|
+
FROM
|
92
|
+
moz_places,
|
93
|
+
moz_bookmarks,
|
94
|
+
moz_items_annos,
|
95
|
+
moz_anno_attributes
|
96
|
+
WHERE
|
97
|
+
moz_anno_attributes.name = 'bookmarkProperties/description' AND
|
98
|
+
moz_items_annos.anno_attribute_id = moz_anno_attributes.id AND
|
99
|
+
moz_items_annos.item_id = moz_bookmarks.id AND
|
100
|
+
moz_places.id = moz_bookmarks.fk AND
|
101
|
+
moz_places.id IN (
|
102
|
+
SELECT DISTINCT fk
|
103
|
+
FROM moz_bookmarks
|
104
|
+
WHERE parent IN (
|
105
|
+
SELECT moz_bookmarks.id
|
106
|
+
FROM moz_bookmarks, moz_bookmarks_roots
|
107
|
+
WHERE moz_bookmarks_roots.root_name = 'tags'
|
108
|
+
AND moz_bookmarks.parent = moz_bookmarks_roots.folder_id
|
109
|
+
)
|
110
|
+
)
|
111
|
+
ORDER BY UPPER(moz_bookmarks.title) ASC
|
112
|
+
SQL
|
113
|
+
|
114
|
+
# tag.id|tag.title
|
115
|
+
QUERY_FOR_TAGS = <<-SQL
|
116
|
+
SELECT id, title FROM moz_bookmarks WHERE parent = 4;
|
117
|
+
SQL
|
118
|
+
|
119
|
+
# all bookmarks with the "given" tag.id
|
120
|
+
QUERY_BY_TAG = <<-SQL
|
121
|
+
SELECT moz_places.id, moz_places.url, moz_places.title, moz_bookmarks.parent
|
122
|
+
FROM moz_places
|
123
|
+
LEFT OUTER JOIN moz_bookmarks
|
124
|
+
ON moz_places.id = moz_bookmarks.fk
|
125
|
+
WHERE moz_bookmarks.parent = @tag_id;
|
126
|
+
SQL
|
127
|
+
|
128
|
+
# bookmark.id|bookmark.fk|bookmark.title
|
129
|
+
# ...
|
130
|
+
#=> 1933|1387|SQLite Home Page
|
131
|
+
# ...
|
132
|
+
QUERY_FOR_SIMPLE_BOOKMARKS = <<-SQL
|
133
|
+
SELECT id,fk,title FROM moz_bookmarks WHERE parent = 2;
|
134
|
+
SQL
|
135
|
+
|
136
|
+
# select the link between bookmars and tags
|
137
|
+
# fk_id = 1387
|
138
|
+
#=> 1934|1387|1144
|
139
|
+
QUERY_FOR_LINK = <<-SQL
|
140
|
+
SELECT id,fk,parent FROM moz_bookmarks WHERE title is null AND fk = @fk_id;
|
141
|
+
SQL
|
142
|
+
|
143
|
+
# tag.id|tag.title
|
144
|
+
# parent = 1144
|
145
|
+
#=> 1144|coding
|
146
|
+
QUERY_FOR_TAG_BY_ID = <<-SQL
|
147
|
+
SELECT id, title FROM moz_bookmarks WHERE parent = 4 AND id = @parent;
|
148
|
+
SQL
|
149
|
+
|
150
|
+
end # QueryConstants
|
151
|
+
|
152
|
+
module HashBookmarksUtil
|
153
|
+
def sort_bookmarks_by(field)
|
154
|
+
if [:tag, :tile].include?(field.intern)
|
155
|
+
self.sort_by do |key, value|
|
156
|
+
value[field.intern]
|
157
|
+
end
|
158
|
+
else
|
159
|
+
self
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end # HashBookmarksUtil
|
data/lib/fundler/version.rb
CHANGED
@@ -2,6 +2,7 @@ require 'fundler_spec_helper'
|
|
2
2
|
|
3
3
|
class TestBookmarksFundler
|
4
4
|
describe Fundler do
|
5
|
+
|
5
6
|
context 'bookmarks functions' do
|
6
7
|
let(:fundler) { Fundler.new }
|
7
8
|
before(:each) do
|
@@ -12,10 +13,21 @@ class TestBookmarksFundler
|
|
12
13
|
FileUtils::rm_rf 'test'
|
13
14
|
end
|
14
15
|
|
15
|
-
it '
|
16
|
+
it 'drop txt bookmarks file' do
|
16
17
|
FileUtils::cd './test' do
|
17
|
-
fundler.
|
18
|
+
fundler.bookmarks(:txt)
|
18
19
|
expect(fundler.all_files).to include('bookmarks_dump.txt')
|
20
|
+
file_line_count = File.foreach('bookmarks_dump.txt').inject(0) { |c, line| c+=1 }
|
21
|
+
expect(fundler.bookmarks_count).to eq (file_line_count/2)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'drop json bookmarks file' do
|
26
|
+
FileUtils::cd './test' do
|
27
|
+
fundler.bookmarks(:json)
|
28
|
+
expect(fundler.all_files).to include('bookmarks_dump.json')
|
29
|
+
file_line_count = File.foreach('bookmarks_dump.json').inject(0) { |c, line| c+=1 }
|
30
|
+
expect(fundler.bookmarks_count).to eq (file_line_count)
|
19
31
|
end
|
20
32
|
end
|
21
33
|
|
@@ -23,6 +35,10 @@ class TestBookmarksFundler
|
|
23
35
|
expect { fundler.get_bookmarks_with_tags }.to_not raise_error
|
24
36
|
end
|
25
37
|
|
38
|
+
it 'get bookmarks url, title, descr and tag' do
|
39
|
+
expect { fundler.get_bookmarks_with_url_title_descr_tag }.to_not raise_error
|
40
|
+
end
|
26
41
|
end # context
|
42
|
+
|
27
43
|
end # describe
|
28
44
|
end # TestBookmarksFundler
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fundler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zeroed
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-05-
|
11
|
+
date: 2013-05-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sqlite3
|