danielsdeleo-teeth 0.0.2 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +11 -0
- data/README.rdoc +107 -10
- data/Rakefile +47 -31
- data/VERSION.yml +4 -0
- data/doc/classes/String.html +182 -0
- data/doc/classes/Teeth/DuplicateDefinitionError.html +113 -0
- data/doc/classes/Teeth/DuplicateRuleError.html +113 -0
- data/doc/classes/Teeth/InvalidDefaultDefinitionName.html +113 -0
- data/doc/classes/Teeth/InvalidExtensionDirectory.html +113 -0
- data/doc/classes/Teeth/RuleStatement.html +291 -0
- data/doc/classes/Teeth/RuleStatementGroup.html +195 -0
- data/doc/classes/Teeth/Scanner.html +535 -0
- data/doc/classes/Teeth/ScannerDefinition.html +253 -0
- data/doc/classes/Teeth/ScannerDefinitionArgumentError.html +113 -0
- data/doc/classes/Teeth/ScannerDefinitionGroup.html +269 -0
- data/doc/classes/Teeth/ScannerError.html +111 -0
- data/doc/classes/Teeth.html +129 -0
- data/doc/created.rid +1 -0
- data/doc/files/README_rdoc.html +314 -0
- data/doc/files/ext/scan_apache_logs/scan_apache_logs_yy_c.html +101 -0
- data/doc/files/ext/scan_rails_logs/scan_rails_logs_yy_c.html +101 -0
- data/doc/files/lib/rule_statement_rb.html +101 -0
- data/doc/files/lib/scanner_definition_rb.html +101 -0
- data/doc/files/lib/scanner_rb.html +108 -0
- data/doc/files/lib/teeth_rb.html +111 -0
- data/doc/fr_class_index.html +39 -0
- data/doc/fr_file_index.html +33 -0
- data/doc/fr_method_index.html +60 -0
- data/doc/index.html +24 -0
- data/doc/rdoc-style.css +208 -0
- data/ext/scan_apache_logs/Makefile +158 -0
- data/ext/scan_apache_logs/extconf.rb +3 -0
- data/ext/scan_apache_logs/scan_apache_logs.yy +267 -0
- data/ext/scan_apache_logs/scan_apache_logs.yy.c +8355 -0
- data/ext/scan_rails_logs/Makefile +158 -0
- data/ext/scan_rails_logs/extconf.rb +3 -0
- data/ext/scan_rails_logs/scan_rails_logs.yy +376 -0
- data/ext/scan_rails_logs/scan_rails_logs.yy.c +11127 -0
- data/lib/rule_statement.rb +61 -0
- data/lib/scanner.rb +98 -0
- data/lib/scanner_definition.rb +116 -0
- data/lib/teeth.rb +5 -1
- data/scanners/scan_apache_logs.rb +27 -0
- data/scanners/scan_rails_logs.rb +70 -0
- data/spec/fixtures/rails_1x.log +59 -0
- data/spec/fixtures/rails_22.log +12 -0
- data/spec/fixtures/rails_22_cached.log +10 -0
- data/spec/fixtures/rails_unordered.log +24 -0
- data/spec/playground/show_apache_processing.rb +13 -0
- data/spec/spec_helper.rb +6 -1
- data/spec/unit/rule_statement_spec.rb +60 -0
- data/spec/unit/{tokenize_apache_spec.rb → scan_apache_spec.rb} +16 -11
- data/spec/unit/scan_rails_logs_spec.rb +90 -0
- data/spec/unit/scaner_definition_spec.rb +65 -0
- data/spec/unit/scanner_spec.rb +109 -0
- data/teeth.gemspec +31 -0
- data/templates/tokenizer.yy.erb +168 -0
- metadata +60 -15
- data/ext/extconf.rb +0 -4
- data/ext/tokenize_apache_logs.yy +0 -215
- data/ext/tokenize_apache_logs.yy.c +0 -12067
@@ -0,0 +1,158 @@
|
|
1
|
+
|
2
|
+
SHELL = /bin/sh
|
3
|
+
|
4
|
+
#### Start of system configuration section. ####
|
5
|
+
|
6
|
+
srcdir = ./
|
7
|
+
topdir = /opt/local/lib/ruby/1.8/i686-darwin8.11.1
|
8
|
+
hdrdir = $(topdir)
|
9
|
+
VPATH = $(srcdir):$(topdir):$(hdrdir)
|
10
|
+
exec_prefix = $(prefix)
|
11
|
+
prefix = $(DESTDIR)/opt/local
|
12
|
+
sharedstatedir = $(prefix)/com
|
13
|
+
mandir = $(DESTDIR)/opt/local/share/man
|
14
|
+
psdir = $(docdir)
|
15
|
+
oldincludedir = $(DESTDIR)/usr/include
|
16
|
+
localedir = $(datarootdir)/locale
|
17
|
+
bindir = $(exec_prefix)/bin
|
18
|
+
libexecdir = $(exec_prefix)/libexec
|
19
|
+
sitedir = $(libdir)/ruby/site_ruby
|
20
|
+
htmldir = $(docdir)
|
21
|
+
vendorarchdir = $(vendorlibdir)/$(sitearch)
|
22
|
+
includedir = $(prefix)/include
|
23
|
+
infodir = $(datarootdir)/info
|
24
|
+
vendorlibdir = $(vendordir)/$(ruby_version)
|
25
|
+
sysconfdir = $(prefix)/etc
|
26
|
+
libdir = $(exec_prefix)/lib
|
27
|
+
sbindir = $(exec_prefix)/sbin
|
28
|
+
rubylibdir = $(libdir)/ruby/$(ruby_version)
|
29
|
+
docdir = $(datarootdir)/doc/$(PACKAGE)
|
30
|
+
dvidir = $(docdir)
|
31
|
+
vendordir = $(DESTDIR)/opt/local/lib/ruby/vendor_ruby
|
32
|
+
datarootdir = $(prefix)/share
|
33
|
+
pdfdir = $(docdir)
|
34
|
+
archdir = $(rubylibdir)/$(arch)
|
35
|
+
sitearchdir = $(sitelibdir)/$(sitearch)
|
36
|
+
datadir = $(datarootdir)
|
37
|
+
localstatedir = $(prefix)/var
|
38
|
+
sitelibdir = $(sitedir)/$(ruby_version)
|
39
|
+
|
40
|
+
CC = /usr/bin/gcc-4.0
|
41
|
+
LIBRUBY = $(LIBRUBY_SO)
|
42
|
+
LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
|
43
|
+
LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
|
44
|
+
LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static
|
45
|
+
|
46
|
+
RUBY_EXTCONF_H =
|
47
|
+
CFLAGS = -fno-common -O2 -fno-common -pipe -fno-common $(cflags) -Wall
|
48
|
+
INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir)
|
49
|
+
DEFS =
|
50
|
+
CPPFLAGS = -I/opt/local/include -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE $(DEFS) $(cppflags)
|
51
|
+
CXXFLAGS = $(CFLAGS)
|
52
|
+
ldflags = -L. -L/opt/local/lib
|
53
|
+
dldflags =
|
54
|
+
archflag =
|
55
|
+
DLDFLAGS = $(ldflags) $(dldflags) $(archflag)
|
56
|
+
LDSHARED = cc -dynamic -bundle -undefined suppress -flat_namespace
|
57
|
+
AR = ar
|
58
|
+
EXEEXT =
|
59
|
+
|
60
|
+
RUBY_INSTALL_NAME = ruby
|
61
|
+
RUBY_SO_NAME = ruby
|
62
|
+
arch = i686-darwin8.11.1
|
63
|
+
sitearch = i686-darwin8.11.1
|
64
|
+
vendorarch = i686-darwin8.11.1
|
65
|
+
ruby_version = 1.8
|
66
|
+
ruby = /opt/local/bin/ruby
|
67
|
+
RUBY = $(ruby)
|
68
|
+
RM = rm -f
|
69
|
+
MAKEDIRS = mkdir -p
|
70
|
+
INSTALL = /usr/bin/install
|
71
|
+
INSTALL_PROG = $(INSTALL) -m 0755
|
72
|
+
INSTALL_DATA = $(INSTALL) -m 644
|
73
|
+
COPY = cp
|
74
|
+
|
75
|
+
#### End of system configuration section. ####
|
76
|
+
|
77
|
+
preload =
|
78
|
+
|
79
|
+
libpath = . $(libdir)
|
80
|
+
LIBPATH = -L. -L$(libdir)
|
81
|
+
DEFFILE =
|
82
|
+
|
83
|
+
CLEANFILES = mkmf.log
|
84
|
+
DISTCLEANFILES =
|
85
|
+
|
86
|
+
extout =
|
87
|
+
extout_prefix =
|
88
|
+
target_prefix = /teeth
|
89
|
+
LOCAL_LIBS =
|
90
|
+
LIBS = $(LIBRUBYARG_SHARED) -lpthread -ldl -lobjc
|
91
|
+
SRCS = scan_rails_logs.yy.c
|
92
|
+
OBJS = scan_rails_logs.yy.o
|
93
|
+
TARGET = scan_rails_logs
|
94
|
+
DLLIB = $(TARGET).bundle
|
95
|
+
EXTSTATIC =
|
96
|
+
STATIC_LIB =
|
97
|
+
|
98
|
+
BINDIR = $(bindir)
|
99
|
+
RUBYCOMMONDIR = $(sitedir)$(target_prefix)
|
100
|
+
RUBYLIBDIR = $(sitelibdir)$(target_prefix)
|
101
|
+
RUBYARCHDIR = $(sitearchdir)$(target_prefix)
|
102
|
+
|
103
|
+
TARGET_SO = $(DLLIB)
|
104
|
+
CLEANLIBS = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map
|
105
|
+
CLEANOBJS = *.o *.a *.s[ol] *.pdb *.exp *.bak
|
106
|
+
|
107
|
+
all: $(DLLIB)
|
108
|
+
static: $(STATIC_LIB)
|
109
|
+
|
110
|
+
clean:
|
111
|
+
@-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES)
|
112
|
+
|
113
|
+
distclean: clean
|
114
|
+
@-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
|
115
|
+
@-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
|
116
|
+
|
117
|
+
realclean: distclean
|
118
|
+
install: install-so install-rb
|
119
|
+
|
120
|
+
install-so: $(RUBYARCHDIR)
|
121
|
+
install-so: $(RUBYARCHDIR)/$(DLLIB)
|
122
|
+
$(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
|
123
|
+
$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
|
124
|
+
install-rb: pre-install-rb install-rb-default
|
125
|
+
install-rb-default: pre-install-rb-default
|
126
|
+
pre-install-rb: Makefile
|
127
|
+
pre-install-rb-default: Makefile
|
128
|
+
$(RUBYARCHDIR):
|
129
|
+
$(MAKEDIRS) $@
|
130
|
+
|
131
|
+
site-install: site-install-so site-install-rb
|
132
|
+
site-install-so: install-so
|
133
|
+
site-install-rb: install-rb
|
134
|
+
|
135
|
+
.SUFFIXES: .c .m .cc .cxx .cpp .C .o
|
136
|
+
|
137
|
+
.cc.o:
|
138
|
+
$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
|
139
|
+
|
140
|
+
.cxx.o:
|
141
|
+
$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
|
142
|
+
|
143
|
+
.cpp.o:
|
144
|
+
$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
|
145
|
+
|
146
|
+
.C.o:
|
147
|
+
$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
|
148
|
+
|
149
|
+
.c.o:
|
150
|
+
$(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $<
|
151
|
+
|
152
|
+
$(DLLIB): $(OBJS)
|
153
|
+
@-$(RM) $@
|
154
|
+
$(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
|
155
|
+
|
156
|
+
|
157
|
+
|
158
|
+
$(OBJS): ruby.h defines.h
|
@@ -0,0 +1,376 @@
|
|
1
|
+
%option prefix="rails_logs_yy"
|
2
|
+
%option full
|
3
|
+
%option never-interactive
|
4
|
+
%option read
|
5
|
+
%option nounput
|
6
|
+
%option noyywrap noreject noyymore nodefault
|
7
|
+
%{
|
8
|
+
#include <ruby.h>
|
9
|
+
#include <uuid/uuid.h>
|
10
|
+
/* Data types */
|
11
|
+
typedef struct {
|
12
|
+
char *key;
|
13
|
+
char *value;
|
14
|
+
} KVPAIR;
|
15
|
+
const KVPAIR EOF_KVPAIR = {"EOF", "EOF"};
|
16
|
+
/* prototypes */
|
17
|
+
char *strip_ends(char *);
|
18
|
+
VALUE t_scan_rails_logs(VALUE);
|
19
|
+
void new_uuid(char *str_ptr);
|
20
|
+
void raise_error_for_string_too_long(VALUE string);
|
21
|
+
void include_message_in_token_hash(VALUE message, VALUE token_hash);
|
22
|
+
void add_uuid_to_token_hash(VALUE token_hash);
|
23
|
+
void push_kv_pair_to_hash(KVPAIR key_value, VALUE token_hash);
|
24
|
+
void concat_word_to_string(KVPAIR key_value, VALUE token_hash);
|
25
|
+
/* Set the scanner name, and return type */
|
26
|
+
#define YY_DECL KVPAIR scan_rails_logs(void)
|
27
|
+
#define yyterminate() return EOF_KVPAIR
|
28
|
+
/* Ruby 1.8 and 1.9 compatibility */
|
29
|
+
#if !defined(RSTRING_LEN)
|
30
|
+
# define RSTRING_LEN(x) (RSTRING(x)->len)
|
31
|
+
# define RSTRING_PTR(x) (RSTRING(x)->ptr)
|
32
|
+
#endif
|
33
|
+
|
34
|
+
%}
|
35
|
+
|
36
|
+
/* Definitions */
|
37
|
+
|
38
|
+
CATCHALL (.|"\n")
|
39
|
+
|
40
|
+
|
41
|
+
WS [[:space:]]
|
42
|
+
|
43
|
+
NON_WS ([a-z]|[0-9]|[:punct:])
|
44
|
+
|
45
|
+
IP4_OCT [0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]
|
46
|
+
|
47
|
+
HOST [a-z0-9][a-z0-9\-]*\.[a-z0-9][a-z0-9\-]*.[a-z0-9][a-z0-9\-\.]*[a-z]+(\:[0-9]+)?
|
48
|
+
|
49
|
+
WDAY mon|tue|wed|thu|fri|sat|sun
|
50
|
+
|
51
|
+
MON jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec
|
52
|
+
|
53
|
+
MONTH_NUM 0[1-9]|1[0-2]
|
54
|
+
|
55
|
+
MDAY 3[0-1]|[1-2][0-9]|0[1-9]
|
56
|
+
|
57
|
+
HOUR 2[0-3]|[0-1][0-9]
|
58
|
+
|
59
|
+
MINSEC [0-5][0-9]|60
|
60
|
+
|
61
|
+
YEAR [0-9][0-9][0-9][0-9]
|
62
|
+
|
63
|
+
PLUSMINUS (\+|\-)
|
64
|
+
|
65
|
+
REL_URL (\/|\\|\.)[a-z0-9\._\~\-\/\?&;#=\%\:\+\[\]\\]*
|
66
|
+
|
67
|
+
PROTO (http:|https:)
|
68
|
+
|
69
|
+
ERR_LVL (emerg|alert|crit|err|error|warn|warning|notice|info|debug)
|
70
|
+
|
71
|
+
HTTP_VERS HTTP\/(1.0|1.1)
|
72
|
+
|
73
|
+
HTTP_VERB (get|head|put|post|delete|trace|connect)
|
74
|
+
|
75
|
+
HTTPCODE (100|101|20[0-6]|30[0-5]|307|40[0-9]|41[0-7]|50[0-5])
|
76
|
+
|
77
|
+
BROWSER_STR \"(moz|msie|lynx).+\"
|
78
|
+
|
79
|
+
RAILS_TEASER (processing|filter\ chain\ halted|rendered)
|
80
|
+
|
81
|
+
CONTROLLER_ACTION [a-z0-9]+#[a-z0-9]+
|
82
|
+
|
83
|
+
RAILS_SKIP_LINES (session\ id)
|
84
|
+
|
85
|
+
CACHE_HIT actioncontroller"::"caching"::"actions"::"actioncachefilter":"0x[0-9a-f]+
|
86
|
+
|
87
|
+
PARTIAL_SESSION_ID ^([a-z0-9]+"="*"-"+[a-z0-9]+)
|
88
|
+
|
89
|
+
RAILS_ERROR_CLASS ([a-z]+\:\:)*[a-z]+error
|
90
|
+
|
91
|
+
%x REQUEST_COMPLETED
|
92
|
+
|
93
|
+
%x COMPLETED_REQ_VIEW_STATS
|
94
|
+
|
95
|
+
%x COMPLETED_REQ_DB_STATS
|
96
|
+
|
97
|
+
|
98
|
+
%%
|
99
|
+
/*
|
100
|
+
Actions
|
101
|
+
*/
|
102
|
+
|
103
|
+
|
104
|
+
{RAILS_TEASER} {
|
105
|
+
KVPAIR teaser = {"teaser", yytext};
|
106
|
+
return teaser;
|
107
|
+
}
|
108
|
+
|
109
|
+
{CONTROLLER_ACTION} {
|
110
|
+
KVPAIR controller_action = {"controller_action", yytext};
|
111
|
+
return controller_action;
|
112
|
+
}
|
113
|
+
|
114
|
+
{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT} {
|
115
|
+
KVPAIR ipv4_addr = {"ipv4_addr", yytext};
|
116
|
+
return ipv4_addr;
|
117
|
+
}
|
118
|
+
|
119
|
+
{YEAR}"-"{MONTH_NUM}"-"{MDAY}{WS}{HOUR}":"{MINSEC}":"{MINSEC} {
|
120
|
+
KVPAIR datetime = {"datetime", yytext};
|
121
|
+
return datetime;
|
122
|
+
}
|
123
|
+
|
124
|
+
{HTTP_VERB} {
|
125
|
+
KVPAIR http_method = {"http_method", yytext};
|
126
|
+
return http_method;
|
127
|
+
}
|
128
|
+
|
129
|
+
{RAILS_SKIP_LINES} {
|
130
|
+
return EOF_KVPAIR;
|
131
|
+
}
|
132
|
+
|
133
|
+
{PARTIAL_SESSION_ID} {
|
134
|
+
KVPAIR end_session_id = {"end_session_id", yytext};
|
135
|
+
return end_session_id;
|
136
|
+
}
|
137
|
+
|
138
|
+
{RAILS_ERROR_CLASS} {
|
139
|
+
KVPAIR error = {"error", yytext};
|
140
|
+
return error;
|
141
|
+
}
|
142
|
+
|
143
|
+
\(({WS}|{NON_WS})+\) {
|
144
|
+
KVPAIR error_message = {"error_message", strip_ends(yytext)};
|
145
|
+
return error_message;
|
146
|
+
}
|
147
|
+
|
148
|
+
"#"[0-9]+{WS} {
|
149
|
+
KVPAIR line_number = {"line_number", strip_ends(yytext)};
|
150
|
+
return line_number;
|
151
|
+
}
|
152
|
+
|
153
|
+
{WS}{REL_URL}":" {
|
154
|
+
KVPAIR file_and_line = {"file_and_line", strip_ends(yytext)};
|
155
|
+
return file_and_line;
|
156
|
+
}
|
157
|
+
|
158
|
+
{CACHE_HIT} {
|
159
|
+
KVPAIR cache_hit = {"cache_hit", yytext};
|
160
|
+
return cache_hit;
|
161
|
+
}
|
162
|
+
|
163
|
+
[a-z0-9]+{REL_URL}/\ \( {
|
164
|
+
KVPAIR partial = {"partial", yytext};
|
165
|
+
return partial;
|
166
|
+
}
|
167
|
+
|
168
|
+
[0-9\.]+/ms\) {
|
169
|
+
KVPAIR render_duration_ms = {"render_duration_ms", yytext};
|
170
|
+
return render_duration_ms;
|
171
|
+
}
|
172
|
+
|
173
|
+
\([0-9\.]+\) {
|
174
|
+
KVPAIR render_duration_s = {"render_duration_s", strip_ends(yytext)};
|
175
|
+
return render_duration_s;
|
176
|
+
}
|
177
|
+
|
178
|
+
completed\ in {
|
179
|
+
BEGIN(REQUEST_COMPLETED);
|
180
|
+
KVPAIR teaser = {"teaser", yytext};
|
181
|
+
return teaser;
|
182
|
+
}
|
183
|
+
|
184
|
+
<REQUEST_COMPLETED>[0-9]+\.[0-9]+ {
|
185
|
+
KVPAIR duration_s = {"duration_s", yytext};
|
186
|
+
return duration_s;
|
187
|
+
}
|
188
|
+
|
189
|
+
<REQUEST_COMPLETED>[0-9]+/ms {
|
190
|
+
KVPAIR duration_ms = {"duration_ms", yytext};
|
191
|
+
return duration_ms;
|
192
|
+
}
|
193
|
+
|
194
|
+
<REQUEST_COMPLETED>(View":"|Rendering":") {
|
195
|
+
BEGIN(COMPLETED_REQ_VIEW_STATS);
|
196
|
+
KVPAIR start_view_stats = {"start_view_stats", yytext};
|
197
|
+
return start_view_stats;
|
198
|
+
}
|
199
|
+
|
200
|
+
<COMPLETED_REQ_VIEW_STATS>([0-9]+\.[0-9]+) {
|
201
|
+
BEGIN(REQUEST_COMPLETED);
|
202
|
+
KVPAIR view_s = {"view_s", yytext};
|
203
|
+
return view_s;
|
204
|
+
}
|
205
|
+
|
206
|
+
<COMPLETED_REQ_VIEW_STATS>[0-9]+ {
|
207
|
+
BEGIN(REQUEST_COMPLETED);
|
208
|
+
KVPAIR view_ms = {"view_ms", yytext};
|
209
|
+
return view_ms;
|
210
|
+
}
|
211
|
+
|
212
|
+
<COMPLETED_REQ_VIEW_STATS>{CATCHALL}
|
213
|
+
|
214
|
+
<REQUEST_COMPLETED>DB":" {
|
215
|
+
BEGIN(COMPLETED_REQ_DB_STATS);
|
216
|
+
KVPAIR start_db_stats = {"start_db_stats", yytext};
|
217
|
+
return start_db_stats;
|
218
|
+
}
|
219
|
+
|
220
|
+
<COMPLETED_REQ_DB_STATS>[0-9]+\.[0-9]+ {
|
221
|
+
BEGIN(REQUEST_COMPLETED);
|
222
|
+
KVPAIR db_s = {"db_s", yytext};
|
223
|
+
return db_s;
|
224
|
+
}
|
225
|
+
|
226
|
+
<COMPLETED_REQ_DB_STATS>[0-9]+ {
|
227
|
+
BEGIN(REQUEST_COMPLETED);
|
228
|
+
KVPAIR db_ms = {"db_ms", yytext};
|
229
|
+
return db_ms;
|
230
|
+
}
|
231
|
+
|
232
|
+
<COMPLETED_REQ_DB_STATS>{CATCHALL}
|
233
|
+
|
234
|
+
<REQUEST_COMPLETED>\[{PROTO}"\/\/"({HOST}|{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT})({REL_URL}|"\/")?\] {
|
235
|
+
KVPAIR url = {"url", strip_ends(yytext)};
|
236
|
+
return url;
|
237
|
+
}
|
238
|
+
|
239
|
+
<REQUEST_COMPLETED>{HTTPCODE} {
|
240
|
+
KVPAIR http_response = {"http_response", yytext};
|
241
|
+
return http_response;
|
242
|
+
}
|
243
|
+
|
244
|
+
<REQUEST_COMPLETED>{NON_WS}{NON_WS}* {
|
245
|
+
KVPAIR strings = {"strings", yytext};
|
246
|
+
return strings;
|
247
|
+
}
|
248
|
+
|
249
|
+
<REQUEST_COMPLETED>{CATCHALL}
|
250
|
+
|
251
|
+
{NON_WS}{NON_WS}* {
|
252
|
+
KVPAIR strings = {"strings", yytext};
|
253
|
+
return strings;
|
254
|
+
}
|
255
|
+
|
256
|
+
{CATCHALL} /* ignore */
|
257
|
+
%%
|
258
|
+
|
259
|
+
char *strip_ends(char *string) {
|
260
|
+
string[yyleng-1] = '\0';
|
261
|
+
++string;
|
262
|
+
return string;
|
263
|
+
}
|
264
|
+
|
265
|
+
void uuid_unparse_upper_sans_dash(const uuid_t uu, char *out)
|
266
|
+
{
|
267
|
+
sprintf(out,
|
268
|
+
"%02X%02X%02X%02X"
|
269
|
+
"%02X%02X"
|
270
|
+
"%02X%02X"
|
271
|
+
"%02X%02X"
|
272
|
+
"%02X%02X%02X%02X%02X%02X",
|
273
|
+
uu[0], uu[1], uu[2], uu[3],
|
274
|
+
uu[4], uu[5],
|
275
|
+
uu[6], uu[7],
|
276
|
+
uu[8], uu[9],
|
277
|
+
uu[10], uu[11], uu[12], uu[13], uu[14], uu[15]);
|
278
|
+
}
|
279
|
+
|
280
|
+
void new_uuid(char *str_ptr){
|
281
|
+
uuid_t new_uuid;
|
282
|
+
uuid_generate_time(new_uuid);
|
283
|
+
uuid_unparse_upper_sans_dash(new_uuid, str_ptr);
|
284
|
+
}
|
285
|
+
|
286
|
+
void raise_error_for_string_too_long(VALUE string){
|
287
|
+
if( RSTRING_LEN(string) > 1000000){
|
288
|
+
rb_raise(rb_eArgError, "string too long for scan_rails_logs! max length is 1,000,000 chars");
|
289
|
+
}
|
290
|
+
}
|
291
|
+
|
292
|
+
/* Scans self, which is expected to be a line from a Rails production or dev log,
|
293
|
+
* and returns a Hash of the significant features in the log message, including
|
294
|
+
* the IP address of the client, the Controller and Action, any partials rendered,
|
295
|
+
* and the time spent rendering them, the duration of the DB request(s), the HTTP
|
296
|
+
* verb, etc. */
|
297
|
+
VALUE t_scan_rails_logs(VALUE self) {
|
298
|
+
KVPAIR kv_result;
|
299
|
+
int scan_complete = 0;
|
300
|
+
int building_words_to_string = 0;
|
301
|
+
VALUE token_hash = rb_hash_new();
|
302
|
+
|
303
|
+
BEGIN(INITIAL);
|
304
|
+
|
305
|
+
/* error out on absurdly large strings */
|
306
|
+
raise_error_for_string_too_long(self);
|
307
|
+
/* {:message => self()} */
|
308
|
+
include_message_in_token_hash(self, token_hash);
|
309
|
+
/* {:id => UUID} */
|
310
|
+
add_uuid_to_token_hash(token_hash);
|
311
|
+
yy_scan_string(RSTRING_PTR(self));
|
312
|
+
while (scan_complete == 0) {
|
313
|
+
kv_result = scan_rails_logs();
|
314
|
+
if (kv_result.key == "EOF"){
|
315
|
+
scan_complete = 1;
|
316
|
+
}
|
317
|
+
else if (kv_result.key == "strings"){
|
318
|
+
/* build a string until we get a non-word */
|
319
|
+
if (building_words_to_string == 0){
|
320
|
+
building_words_to_string = 1;
|
321
|
+
push_kv_pair_to_hash(kv_result, token_hash);
|
322
|
+
}
|
323
|
+
else{
|
324
|
+
concat_word_to_string(kv_result, token_hash);
|
325
|
+
}
|
326
|
+
}
|
327
|
+
else {
|
328
|
+
building_words_to_string = 0;
|
329
|
+
push_kv_pair_to_hash(kv_result, token_hash);
|
330
|
+
}
|
331
|
+
}
|
332
|
+
yy_delete_buffer(YY_CURRENT_BUFFER);
|
333
|
+
return rb_obj_dup(token_hash);
|
334
|
+
}
|
335
|
+
|
336
|
+
void add_uuid_to_token_hash(VALUE token_hash) {
|
337
|
+
char new_uuid_str[33];
|
338
|
+
new_uuid(new_uuid_str);
|
339
|
+
VALUE hsh_key_id = ID2SYM(rb_intern("id"));
|
340
|
+
VALUE hsh_val_id = rb_tainted_str_new2(new_uuid_str);
|
341
|
+
rb_hash_aset(token_hash, hsh_key_id, hsh_val_id);
|
342
|
+
}
|
343
|
+
|
344
|
+
void include_message_in_token_hash(VALUE message, VALUE token_hash) {
|
345
|
+
/* {:message => self()} */
|
346
|
+
VALUE hsh_key_msg = ID2SYM(rb_intern("message"));
|
347
|
+
rb_hash_aset(token_hash, hsh_key_msg, message);
|
348
|
+
}
|
349
|
+
|
350
|
+
void concat_word_to_string(KVPAIR key_value, VALUE token_hash) {
|
351
|
+
char * space = " ";
|
352
|
+
VALUE hsh_key = ID2SYM(rb_intern(key_value.key));
|
353
|
+
VALUE hsh_value = rb_hash_aref(token_hash, hsh_key);
|
354
|
+
VALUE string = rb_ary_entry(hsh_value, -1);
|
355
|
+
rb_str_cat(string, space, 1);
|
356
|
+
rb_str_cat(string, key_value.value, yyleng);
|
357
|
+
}
|
358
|
+
|
359
|
+
void push_kv_pair_to_hash(KVPAIR key_value, VALUE token_hash) {
|
360
|
+
VALUE hsh_key = ID2SYM(rb_intern(key_value.key));
|
361
|
+
VALUE hsh_value = rb_hash_aref(token_hash, hsh_key);
|
362
|
+
VALUE ary_for_token_type = rb_ary_new();
|
363
|
+
switch (TYPE(hsh_value)) {
|
364
|
+
case T_NIL:
|
365
|
+
rb_ary_push(ary_for_token_type, rb_tainted_str_new2(key_value.value));
|
366
|
+
rb_hash_aset(token_hash, hsh_key, ary_for_token_type);
|
367
|
+
break;
|
368
|
+
case T_ARRAY:
|
369
|
+
rb_ary_push(hsh_value, rb_tainted_str_new2(key_value.value));
|
370
|
+
break;
|
371
|
+
}
|
372
|
+
}
|
373
|
+
|
374
|
+
void Init_scan_rails_logs() {
|
375
|
+
rb_define_method(rb_cString, "scan_rails_logs", t_scan_rails_logs, 0);
|
376
|
+
}
|