feedparser 0.2.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Manifest.txt +9 -2
- data/README.md +69 -27
- data/Rakefile +2 -1
- data/lib/feedparser.rb +7 -1
- data/lib/feedparser/builder/atom.rb +92 -42
- data/lib/feedparser/builder/rss.rb +70 -42
- data/lib/feedparser/feed.rb +7 -14
- data/lib/feedparser/item.rb +9 -12
- data/lib/feedparser/parser.rb +3 -2
- data/lib/feedparser/version.rb +3 -3
- data/test/feeds/googlegroups.atom +19 -0
- data/test/feeds/googlegroups2.atom +7 -0
- data/test/feeds/headius.atom +122 -0
- data/test/feeds/lambdatheultimate.rss2 +288 -0
- data/test/feeds/railstutorial.atom +655 -0
- data/test/feeds/rubyflow.rss2 +116 -0
- data/test/feeds/rubymine.rss2 +315 -0
- data/test/feeds/rubyonrails.atom +1241 -0
- data/test/feeds/sitepoint.rss2 +219 -0
- data/test/helper.rb +36 -9
- data/test/test_atom.rb +14 -32
- data/test/test_atom_live.rb +45 -0
- data/test/test_rss.rb +13 -25
- data/test/test_rss_live.rb +38 -0
- metadata +27 -5
- data/test/feeds/quirksblog.atom.v03 +0 -1098
- data/test/test_atom_from_file.rb +0 -28
data/lib/feedparser/feed.rb
CHANGED
|
@@ -1,40 +1,33 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
|
|
1
3
|
module FeedParser
|
|
2
4
|
|
|
3
5
|
class Feed
|
|
4
|
-
### attr_accessor :object # not use for now
|
|
5
6
|
|
|
6
7
|
attr_accessor :format # e.g. atom|rss 2.0|etc.
|
|
7
8
|
attr_accessor :title
|
|
8
|
-
attr_accessor :title_type # e.g. text|html|html-escaped (optional) -use - why?? why not??
|
|
9
9
|
attr_accessor :url
|
|
10
10
|
|
|
11
11
|
attr_accessor :items
|
|
12
12
|
|
|
13
13
|
def summary?() @summary.nil? == false; end
|
|
14
|
-
attr_accessor :summary # e.g. description (rss)
|
|
15
|
-
attr_accessor :summary_type # e.g. text|html|html-escaped
|
|
14
|
+
attr_accessor :summary # e.g. description (rss)|subtitle (atom)
|
|
16
15
|
|
|
17
|
-
def
|
|
18
|
-
attr_accessor :
|
|
19
|
-
attr_accessor :title2_type # e.g. text|html|html-escaped
|
|
16
|
+
def updated?() @updated.nil? == false; end
|
|
17
|
+
attr_accessor :updated # e.g. lastBuildDate (rss)|updated (atom)
|
|
20
18
|
|
|
21
19
|
def published?() @published.nil? == false; end
|
|
22
|
-
attr_accessor :published
|
|
23
|
-
|
|
24
|
-
def updated?() @updated.nil? == false; end
|
|
25
|
-
attr_accessor :updated
|
|
20
|
+
attr_accessor :published # e.g. pubDate (rss)\n/a (atom) -- note: published is basically an alias for created
|
|
26
21
|
|
|
27
|
-
def built?() @built.nil? == false; end
|
|
28
|
-
attr_accessor :built
|
|
29
22
|
|
|
30
23
|
attr_accessor :generator
|
|
31
24
|
attr_accessor :generator_version # e.g. @version (atom)
|
|
32
25
|
attr_accessor :generator_uri # e.g. @uri (atom) - use alias url/link ???
|
|
33
26
|
|
|
27
|
+
|
|
34
28
|
## fix:
|
|
35
29
|
# add pretty printer/inspect (exclude object)
|
|
36
30
|
|
|
37
|
-
|
|
38
31
|
end # class Feed
|
|
39
32
|
|
|
40
33
|
end # module FeedParser
|
data/lib/feedparser/item.rb
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
|
|
1
3
|
module FeedParser
|
|
2
4
|
|
|
3
5
|
class Item
|
|
4
|
-
## attr_accessor :object # not used for now -- orginal object (e.g RSS item or ATOM entry etc.)
|
|
5
6
|
|
|
6
7
|
attr_accessor :title
|
|
7
|
-
attr_accessor :
|
|
8
|
-
attr_accessor :url # todo: rename to link (use alias) ??
|
|
9
|
-
|
|
8
|
+
attr_accessor :url
|
|
10
9
|
|
|
11
10
|
def content?() @content.nil? == false; end
|
|
12
11
|
attr_accessor :content
|
|
@@ -14,20 +13,18 @@ class Item
|
|
|
14
13
|
|
|
15
14
|
def summary?() @summary.nil? == false; end
|
|
16
15
|
attr_accessor :summary
|
|
17
|
-
attr_accessor :summary_type # optional for now (text|html|html-escaped) - not yet set
|
|
18
|
-
|
|
19
|
-
## todo: add summary (alias description) ???
|
|
20
|
-
## todo: add author/authors
|
|
21
|
-
## todo: add category/categories
|
|
22
16
|
|
|
23
|
-
def published?() @published.nil? == false; end
|
|
24
|
-
attr_accessor :published
|
|
25
17
|
|
|
26
18
|
def updated?() @updated.nil? == false; end
|
|
27
|
-
attr_accessor :updated
|
|
19
|
+
attr_accessor :updated # pubDate (RSS)|updated (Atom)
|
|
20
|
+
|
|
21
|
+
def published?() @published.nil? == false; end
|
|
22
|
+
attr_accessor :published # note: published is basically an alias for created
|
|
28
23
|
|
|
29
24
|
attr_accessor :guid # todo: rename to id (use alias) ??
|
|
30
25
|
|
|
26
|
+
## todo: add author/authors
|
|
27
|
+
## todo: add category/categories
|
|
31
28
|
|
|
32
29
|
end # class Item
|
|
33
30
|
|
data/lib/feedparser/parser.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
1
2
|
|
|
2
3
|
module FeedParser
|
|
3
4
|
|
|
@@ -25,7 +26,7 @@ class Parser
|
|
|
25
26
|
parser.do_validate = false
|
|
26
27
|
parser.ignore_unknown_element = true
|
|
27
28
|
|
|
28
|
-
|
|
29
|
+
logger.debug "Parsing feed..."
|
|
29
30
|
feed_wild = parser.parse # not yet normalized
|
|
30
31
|
|
|
31
32
|
logger.debug " feed.class=#{feed_wild.class.name}"
|
|
@@ -36,7 +37,7 @@ class Parser
|
|
|
36
37
|
feed = RssFeedBuilder.build( feed_wild )
|
|
37
38
|
end
|
|
38
39
|
|
|
39
|
-
|
|
40
|
+
logger.debug "== #{feed.format} / #{feed.title} =="
|
|
40
41
|
feed # return new (normalized) feed
|
|
41
42
|
end
|
|
42
43
|
|
data/lib/feedparser/version.rb
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module FeedParser
|
|
4
4
|
|
|
5
|
-
MAJOR =
|
|
6
|
-
MINOR =
|
|
5
|
+
MAJOR = 1
|
|
6
|
+
MINOR = 0
|
|
7
7
|
PATCH = 0
|
|
8
8
|
VERSION = [MAJOR,MINOR,PATCH].join('.')
|
|
9
9
|
|
|
@@ -13,7 +13,7 @@ module FeedParser
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
def self.banner
|
|
16
|
-
"feedparser/#{VERSION} (with
|
|
16
|
+
"feedparser/#{VERSION} (with rss/#{RSS::VERSION}) on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
def self.root
|
|
@@ -16,3 +16,22 @@
|
|
|
16
16
|
<summary type="html">Hello, I've started putting together a planet site for beer, that is, Planet Beer [1]. The first feed lists include: - Austria [2] - Belgium [3] You're welcome and invited to suggest new countries and feeds. Cheers. Prost. [1] http://planetbeer.herokuapp.com [2] http://github.com/openbeer/planet/blob/master/</summary>
|
|
17
17
|
</entry>
|
|
18
18
|
</feed>
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
feed.format: atom
|
|
23
|
+
feed.generator: Google Groups
|
|
24
|
+
feed.title: Open Beer & Brewery Database (beer.db)
|
|
25
|
+
feed.url: https://groups.google.com/d/forum/beerdb
|
|
26
|
+
|
|
27
|
+
feed.items[0].title: Planet Beer (Austria, Belgium) - Feeds Incl. Craft Fest Wien, Beer-A-Day, proBier n Friends
|
|
28
|
+
feed.items[0].url: https://groups.google.com/d/topic/beerdb/KpQOUDYJ3J8
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
### todo: fix: &amp; => & -> always assume plain text? (by default) - auto-escape xml entities??
|
|
32
|
+
feed.summary: Free open public domain beer database & schema (beer.db) for use in any (programming) language (e.g. uses plain text fixtures/data sets). Questions? Comments?
|
|
33
|
+
|
|
34
|
+
### todo: add check for datetime (use to_s ??)
|
|
35
|
+
## feed.updated.to_s: 2014-12-31T15:33:00+00:00
|
|
36
|
+
## feed.items[0].to_s: 2014-12-31T15:33:00+00:00
|
|
37
|
+
|
|
@@ -18,3 +18,10 @@
|
|
|
18
18
|
<summary type="html">Hello, I've started putting together a planet site for beer, that is, Planet Beer [1]. The first feed lists include: - Austria [2] - Belgium [3] You're welcome and invited to suggest new countries and feeds. Cheers. Prost. [1] http://planetbeer.herokuapp.com [2] http://github.com/openbeer/planet/blob/master/</summary>
|
|
19
19
|
</entry>
|
|
20
20
|
</feed>
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
feed.format: atom
|
|
25
|
+
feed.generator: Google Groups (w/ leading n trailing newlines stripped)
|
|
26
|
+
feed.url: https://groups.google.com/d/forum/beerdb
|
|
27
|
+
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
<?xml version='1.0' encoding='UTF-8'?>
|
|
2
|
+
<?xml-stylesheet href="http://www.blogger.com/styles/atom.css" type="text/css"?>
|
|
3
|
+
<feed
|
|
4
|
+
xmlns='http://www.w3.org/2005/Atom'
|
|
5
|
+
xmlns:openSearch='http://a9.com/-/spec/opensearchrss/1.0/'
|
|
6
|
+
xmlns:blogger='http://schemas.google.com/blogger/2008'
|
|
7
|
+
xmlns:georss='http://www.georss.org/georss'
|
|
8
|
+
xmlns:gd="http://schemas.google.com/g/2005"
|
|
9
|
+
xmlns:thr='http://purl.org/syndication/thread/1.0'>
|
|
10
|
+
<id>tag:blogger.com,1999:blog-4704664917418794835</id>
|
|
11
|
+
<updated>2015-01-16T15:31:29.613-08:00</updated>
|
|
12
|
+
<category term="jruby"/>
|
|
13
|
+
<category term="ruby"/>
|
|
14
|
+
<category term="java"/>
|
|
15
|
+
<category term="rails"/>
|
|
16
|
+
<category term="netbeans"/>
|
|
17
|
+
<category term="invokedynamic"/>
|
|
18
|
+
<category term="javapolis"/>
|
|
19
|
+
<category term="jvm"/>
|
|
20
|
+
<category term="jython"/>
|
|
21
|
+
<category term="python"/><category term="sun"/><category term="yarv"/><category term="applet"/><category term="application bundle"/><category term="compilation"/><category term="compiler"/><category term="dynamic dispatch"/><category term="dynamic languages"/><category term="eclipse"/><category term="enterprise"/><category term="file structure"/><category term="glassfish"/>
|
|
22
|
+
<category term="gpl"/><category term="grails"/><category term="groovy"/><category term="gsoc"/><category term="humor"/><category term="irb"/><category term="ironruby"/><category term="jdk"/><category term="jruby on rails"/><category term="jruby release"/><category term="jsr223"/><category term="jsr270"/><category term="jsr292"/><category term="keywords"/>
|
|
23
|
+
<category term="macruby"/><category term="magic"/><category term="mcgovern"/><category term="meetup"/><category term="methods"/><category term="mongrel"/><category term="open source"/><category term="optimization"/><category term="os x"/><category term="presentation"/><category term="programming languages"/><category term="quick outline"/><category term="rubinius"/>
|
|
24
|
+
<category term="ruby 2.0"/><category term="ruby compiler"/><category term="scripting"/><category term="tech days"/><category term="tiobe"/>
|
|
25
|
+
<title type='text'>Headius</title>
|
|
26
|
+
<subtitle type='html'>Helping the JVM Into the 21st Century</subtitle>
|
|
27
|
+
<link rel='http://schemas.google.com/g/2005#feed' type='application/atom+xml' href='http://blog.headius.com/feeds/posts/default'/>
|
|
28
|
+
<link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default'/>
|
|
29
|
+
<link rel='alternate' type='text/html' href='http://blog.headius.com/'/>
|
|
30
|
+
<link rel='hub' href='http://pubsubhubbub.appspot.com/'/>
|
|
31
|
+
<link rel='next' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default?start-index=26&max-results=25'/>
|
|
32
|
+
<author>
|
|
33
|
+
<name>Charles Nutter</name>
|
|
34
|
+
<uri>https://plus.google.com/101599370339210456684</uri>
|
|
35
|
+
<email>noreply@blogger.com</email>
|
|
36
|
+
<gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/>
|
|
37
|
+
</author>
|
|
38
|
+
<generator version='7.00' uri='http://www.blogger.com'>Blogger</generator>
|
|
39
|
+
<openSearch:totalResults>315</openSearch:totalResults>
|
|
40
|
+
<openSearch:startIndex>1</openSearch:startIndex>
|
|
41
|
+
<openSearch:itemsPerPage>25</openSearch:itemsPerPage>
|
|
42
|
+
<entry>
|
|
43
|
+
<id>tag:blogger.com,1999:blog-4704664917418794835.post-3430080308857860963</id>
|
|
44
|
+
<published>2014-05-21T10:44:00.000-07:00</published>
|
|
45
|
+
<updated>2014-05-21T10:44:01.683-07:00</updated>
|
|
46
|
+
<title type='text'>JRubyConf.eu 2014!</title>
|
|
47
|
+
<content type='html'>
|
|
48
|
+
<div dir="ltr" style="text-align: left;"
|
|
49
|
+
trbidi="on">I'm thrilled to announce that we'll have another edition
|
|
50
|
+
of <a href="http://2014.jrubyconf.eu/">JRubyConf.eu</a>&nbsp;this year!<br /><br /><a href="http://2013.jrubyconf.eu/">
|
|
51
|
+
Last year's event</a> was a great success.
|
|
52
|
+
We had a two-day conference in Berlin immediately before <a href="http://2013.eurucamp.org/">Eurucamp 2013</a>, with two speakers from the core team (myself and <a href="http://twitter.com/tom_enebo">Tom Enebo</a>) and a whopping <b>fifteen</b>&nbsp;non-core speakers. A great event was had by all.<br /><br />This year, we've decided to pull the event back to its roots, as part of <a href="http://2014.eurucamp.org/">Eurucamp 2014</a>. We'll return to the single-track, single-day event co-located with and immediately preceding Eurucamp on 1st August. We really wanted to bring JRuby back to Rubyists, and we're looking forward to hanging out at Eurucamp the whole weekend!<br /><br />Why not visit Eurucamp early and spend a day learning about JRuby with the best JRubyists in Europe?<br /><br />If you're interested in attending, tickets are available for only €99 at the <a href="http://tickets.eurucamp.org/">Eurucamp ticket site</a> now!<br /><br />We're also looking for speakers from the JRuby community. You can submit to the CFP (which ends Sunday 28 May) using the <a href="http://cfp.eurucamp.org/">Eurucamp CFP app</a>.<br /><br />Looking forward to seeing you at JRubyConf and Eurucamp this summer!</div>
|
|
53
|
+
</content>
|
|
54
|
+
<link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/3430080308857860963/comments/default' title='Post Comments'/>
|
|
55
|
+
<link rel='replies' type='text/html' href='http://blog.headius.com/2014/05/jrubyconfeu-2014.html#comment-form' title='0 Comments'/>
|
|
56
|
+
<link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/3430080308857860963'/>
|
|
57
|
+
<link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/3430080308857860963'/>
|
|
58
|
+
<link rel='alternate' type='text/html' href='http://blog.headius.com/2014/05/jrubyconfeu-2014.html' title='JRubyConf.eu 2014!'/>
|
|
59
|
+
<author>
|
|
60
|
+
<name>Charles Nutter</name>
|
|
61
|
+
<uri>https://plus.google.com/101599370339210456684</uri>
|
|
62
|
+
<email>noreply@blogger.com</email>
|
|
63
|
+
<gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/>
|
|
64
|
+
</author>
|
|
65
|
+
<thr:total>0</thr:total>
|
|
66
|
+
</entry>
|
|
67
|
+
<entry>
|
|
68
|
+
<id>tag:blogger.com,1999:blog-4704664917418794835.post-462657466694269626</id>
|
|
69
|
+
<published>2013-06-07T01:58:00.002-07:00</published>
|
|
70
|
+
<updated>2013-06-07T02:00:53.773-07:00</updated>
|
|
71
|
+
<title type='text'>The Pain of Broken Subprocess Management on JDK</title>
|
|
72
|
+
<content type='html'><script type="text/javascript">SyntaxHighlighter.defaults.gutter = false;</script><div dir="ltr" style="text-align: left;" trbidi="on">I prefer to write happy posts...I really do. But tonight I'm completely defeated by the JDK's implementation of subprocess launching, and I need to tell the world why.<br /><br />JRuby has always strived to mimic MRI's behavior as much as possible, which in many cases has meant we need to route around the JDK to get at true POSIX APIs and behaviors.<br /><br />For example, JRuby has provided the ability to manipulate symbolic links since well before Java 7 provided that capability, using a native POSIX subsystem built atop jnr-ffi, our Java-to-C FFI layer (courtesy of Wayne Meissner). Everyone in the Java world knew for years the lack of symlink support was a gross omission, but most folks just sucked it up and went about their business. We could not afford to do that.<br /><br />We've repeated this process for many other Ruby features: UNIX sockets, libc-like IO, selectable stdin, filesystem attributes...on and on. And we've been able to provide the best POSIX runtime on the JVM <b>bar none</b>. Nobody has gone as far or done as much as JRuby has.<br /><br />Another area where we've had to route around the JDK is in subprocess launching and management. The JDK provides java.lang.ProcessBuilder, an API for assembling the appropriate pieces of a subprocess launch, producing a java.lang.Process object. Process in turn provides methods to wait for the subprocess, get access to its streams, and destroy it forcibly. It works great, on the surface.<br /><br />Unfortunately, the cake is a lie.<br /><br />Under the covers, the JDK implements Process through a complicated series of tricks. We want to be able to interactively control the child process, monitor it for writes, govern its lifecycle exactly. The JDK attempts to provide a consistent experience across all platforms. Unfortunately, those two worlds are not currently compatible, and the resulting experience is consistently awful.<br /><br />We'll start at the bottom to see where things go wrong.<br /><br /><h4 style="text-align: left;">POSIX, POSIX, Everywhere</h4><br />At the core of ProcessBuilder, inside the native code behind UNIXProcess, we do find somewhat standard POSIX calls to fork and exec, wrapped up in a native downcall forkAndExec:<br /><br /><div><script class="brush: java;" type="syntaxhighlighter"><![CDATA[ /** * Create a process using fork(2) and exec(2). * * @param std_fds array of file descriptors. Indexes 0, 1, and * 2 correspond to standard input, standard output and * standard error, respectively. On input, a value of -1 * means to create a pipe to connect child and parent * processes. On output, a value which is not -1 is the * parent pipe fd corresponding to the pipe which has * been created. An element of this array is -1 on input * if and only if it is <em>not</em> -1 on output. * @return the pid of the subprocess */ private native int forkAndExec(byte[] prog, byte[] argBlock, int argc, byte[] envBlock, int envc, byte[] dir, int[] std_fds, boolean redirectErrorStream) ]]></script><br /></div>The C code behind this is a bit involved, so I'll summarize what it does.<br /><ol style="text-align: left;"><li>Sets up pipes for in, out, err, and fail to communicate with the eventual child process.</li><li>Copies the parent's descriptors from the pipes into the "fds" array.</li><li>Launches the child through a fairly standard fork+exec sequence.</li><li>Waits for the child to write a byte to the fail pipe indicating success or failure.</li><li>Scrubs the unused sides of the pipes in parent and child.</li><li>Returns the child process ID.</li></ol><div>This is all pretty standard for subprocess launching, and if it proceeded to put those file descriptors into direct, selectable channels we'd have no issues. Unfortunately, things immediately go awry once we return to the Java code.<br /><br /><h4 style="text-align: left;">Interactive?</h4><br />The call to forkAndExec occurs inside the UNIXProcess constructor, as the very first thing it does. At that point, it has in hand the three standard file descriptors and the subprocess pid, and it knows that the subprocess has at least been successfully forked. The next step is to wrap the file descriptors in appropriate InputStream and OutputStream objects, and this is where we find the first flaw.<br /><br /><div><script class="brush: java;" type="syntaxhighlighter"><![CDATA[ if (std_fds[0] == -1) stdin_stream = ProcessBuilder.NullOutputStream.INSTANCE; else { FileDescriptor stdin_fd = new FileDescriptor(); fdAccess.set(stdin_fd, std_fds[0]); stdin_stream = new BufferedOutputStream( new FileOutputStream(stdin_fd)); } ]]></script><br /></div>This is the code to set up an OutputStream for the input channel of the child process, so we can write to it. Now we know the operating system is going to funnel those written bytes directly to the subprocess's input stream, and ideally if we're launching a subprocess we intend to control it...perhaps by sending it interactive commands. Why, then, do we wrap the file descriptor with a BufferedOutputStream? <br />This is where JRuby's hacks begin. In our process subsystem, we have the following piece of code, which attempts to unwrap buffering from any stream it is given. <br /><br /><div><script class="brush: java;" type="syntaxhighlighter"><![CDATA[ /** * Unwrap all filtering streams between the given stream and its actual * unfiltered stream. This is primarily to unwrap streams that have * buffers that would interfere with interactivity. * * @param filteredStream The stream to unwrap * @return An unwrapped stream, presumably unbuffered */ public static OutputStream unwrapBufferedStream(OutputStream filteredStream) { if (RubyInstanceConfig.NO_UNWRAP_PROCESS_STREAMS) return filteredStream; while (filteredStream instanceof FilterOutputStream) { try { filteredStream = (OutputStream) FieldAccess.getProtectedFieldValue(FilterOutputStream.class, "out", filteredStream); } catch (Exception e) { break; // break out if we've dug as deep as we can } } return filteredStream; } ]]></script></div><br />The FieldAccess.getProtectedFieldValue call there does what you think it does...attempt to read the "out" field from within FilteredOutputStream, which in this case will be the FileOutputStream from above. Unwrapping the stream in this way allows us to do two things:<br /><ol style="text-align: left;"><li>We can do unbuffered writes to (or reads from, in the case of the child's out and err streams) the child process.</li><li>We can get access to the more direct FileChannel for the stream, to do direct ByteBuffer reads and writes or low-level stream copying.</li></ol><div>So we're in good shape, right? It's a bit of hackery, but we've got our unbuffered Channel and can interact directly with the subprocess. Is this good enough?</div><div><br /></div><div>I wish it were.<br /><br /></div><h4 style="text-align: left;">Selectable?</h4><div><br />The second problem we run into is that users very often would like to select against the output streams of the child process, to perform nonblocking IO operations until the child has actually written some data. It gets reported as a JRuby bug over and over again because there's simply no way for us to implement it. Why? Because FileChannel is not selectable. <br /><br /><script class="brush: java;" type="syntaxhighlighter"><![CDATA[ public abstract class FileChannel extends AbstractInterruptibleChannel implements SeekableByteChannel, GatheringByteChannel, ScatteringByteChannel ]]></script><br />FileChannel implements methods for random-access reads and writes (positioning) and blocking IO interruption (which NIO implements by closing the stream...that's a rant for another day), but it does not implement any of the logic necessary for doing nonblocking IO using an NIO Selector. This comes up in at least one other place: the JVM's own standard IO streams are also not selectable, which means you can't select for user input at the console. Consistent experience indeed...it seems that all interaction with the user or with processes must be treated as file IO, with no selection capabilities. <br /><br />(It is interesting to note that the JVM's standard IO streams are *also* wrapped in buffers, which we dutifully unwrap to provide a truly interactive console.) <br /><br />Why are inter-proces file descriptors, which would support selector operations just wonderfully, wrapped in an unselectable channel? I have no idea, and it's impossible for us to hack around. <br /><br />Let's not dwell on this item, since there's more to cover. <br /><br /><h4 style="text-align: left;">Fear the Reaper</h4></div></div><div><br /></div><div>You may recall I also wanted to have direct control over the lifecycle of the subprocess, to be able to wait for it or kill it at my own discretion. And on the surface, Process appears to provide these capabilities via the waitFor() and destroy() methods. Again it's all smoke and mirrors.</div><div><br /></div><div>Further down in the UNIXProcess constructor, you'll find this curious piece of code:</div><div><br /></div><div><script class="brush: java;" type="syntaxhighlighter"><![CDATA[ /* * For each subprocess forked a corresponding reaper thread * is started. That thread is the only thread which waits * for the subprocess to terminate and it doesn't hold any * locks while doing so. This design allows waitFor() and * exitStatus() to be safely executed in parallel (and they * need no native code). */ java.security.AccessController.doPrivileged( new java.security.PrivilegedAction<void>() { public Void run() { Thread t = new Thread("process reaper") { public void run() { int res = waitForProcessExit(pid); synchronized (UNIXProcess.this) { hasExited = true; exitcode = res; UNIXProcess.this.notifyAll(); } } }; t.setDaemon(true); t.start(); return null; }}); ]]></script> <div><br /></div><div>For each subprocess started through this API, the JVM will spin up a "process reaper" thread. This thread is designed to monitor the subprocess for liveness and notify the parent UNIXProcess object when that process has died, so it can pass on that information to the user via the waitFor() and exitValue() API calls.</div><div><br /></div><div>The interesting bit here is the waitForProcessExit(pid) call, which is another native downcall into C land:</div><div><br /><script class="brush: c;" type="syntaxhighlighter"><![CDATA[ /* Block until a child process exits and return its exit code. Note, can only be called once for any given pid. */ JNIEXPORT jint JNICALL Java_java_lang_UNIXProcess_waitForProcessExit(JNIEnv* env, jobject junk, jint pid) { /* We used to use waitid() on Solaris, waitpid() on Linux, but * waitpid() is more standard, so use it on all POSIX platforms. */ int status; /* Wait for the child process to exit. This returns immediately if the child has already exited. */ while (waitpid(pid, &status, 0) < 0) { switch (errno) { case ECHILD: return 0; case EINTR: break; default: return -1; } } if (WIFEXITED(status)) { /* * The child exited normally; get its exit code. */ return WEXITSTATUS(status); } else if (WIFSIGNALED(status)) { /* The child exited because of a signal. * The best value to return is 0x80 + signal number, * because that is what all Unix shells do, and because * it allows callers to distinguish between process exit and * process death by signal. * Unfortunately, the historical behavior on Solaris is to return * the signal number, and we preserve this for compatibility. */ #ifdef __solaris__ return WTERMSIG(status); #else return 0x80 + WTERMSIG(status); #endif } else { /* * Unknown exit code; pass it through. */ return status; } } ]]></script><br /><div><br /></div><div>There's nothing too peculiar here; this is how you'd wait for the child process to exit if you were writing plain old C code. But there's a sinister detail you can't see just by looking at this code: waitpid can be called <b>exactly once</b>&nbsp;by the parent process.</div><div><br /></div><div>Part of the Ruby Process API is the ability to get a subprocess PID and wait for it. The concept of a process ID has been around for a long time, and Rubyists (even amateur Rubyists who've never written a line of C code) don't seem to have any problem calling Process.waitpid when they want to wait for a child to exit. JRuby is an implementation of Ruby, and we would ideally like to be able to run all Ruby code that exists, so we also must implement Process.waitpid in some reasonable way. Our choice was to literally call the C function waitpid(2) via our FFI layer.</div><div><br /></div><div>Here's the subtle language from the wait(2) manpage (which includes waitpid):</div><div><pre><br />RETURN VALUES<br /> If wait() returns due to a stopped or terminated child<br /> process, the process ID of the child is returned to the<br /> calling process. Otherwise, a value of -1 is returned<br /> and errno is set to indicate the error.<br /><br /> If wait3(), wait4(), or waitpid() returns due to a<br /> stopped or terminated child process, the process ID of<br /> the child is returned to the calling process. If there<br /> are no children not previously awaited, -1 is returned<br /> with errno set to [ECHILD]. Otherwise, if WNOHANG is<br /> specified and there are no stopped or exited children,<br /> 0 is returned. If an error is detected or a caught<br /> signal aborts the call, a value of -1 is returned and<br /> errno is set to indicate the error.<br /></pre></div><div>There's a lot of negatives and passives and conditions there, so I'll spell it out for you more directly: If you call waitpid for a given child PID and someone else in your process has already done so...bad things happen.</div><div><br /></div><div>We effectively have to race the JDK to the waitpid call. If we get there first, the reaper thread bails out immediately and does no further work. If we don't get their first, it becomes impossible for a Ruby user to waitpid for that child process.</div><div><br /></div><div>Now you may be saying "why don't you just wait on the Process object and let the JDK do its job, old man? The problem here is that Ruby's Process API behaves like a POSIX process API: you get a PID back, and you wait on that PID. We can't mimic that API without returning a PID and implementing Process.waitpid appropriately.</div><div><br /></div><div>(Interesting note: we also use reflection tricks to get the real PID out of the java.lang.Process object, since it is not normally exposed.)</div><div><br /></div><div>Could we have some internal lookup table mapping PIDs to Process objects, and make our wait logic just call Process.waitFor? In order to do so, we'd need to manage a weak-valued map from integers to Process objects...which is certainly doable, but it breaks if someone uses a native library or FFI call to launch a process themselves. Oh, but if it's not in our table we could do waitpid. And so the onion grows more layers, all because we can't simply launch a process, get a PID, and wait on it.</div><div><br /></div><div>It doesn't end here, though.</div><div><br /></div><h4 style="text-align: left;">Keep Boiling That Ocean</h4><div><br /></div><div>At this point we've managed to at least get interactive streams to the child process, and even if they're not selectable that's a big improvement over the standard API. We've managed to dig out a process ID and sometimes we can successfully wait for it with a normal waitpid function call. So out of our three goals (interactivity, selectability, lifecycle control) we're maybe close to halfway there.</div><div><br /></div><div>Then the JDK engineers go and pull the rug out from under us.</div><div><br /></div><div>The logic for UNIXProcess has changed over time. Here's the notable differences in the current JDK 7 codebase:</div><div><ul style="text-align: left;"><li>An Executor is now used to avoid spinning up a new thread for each child process. I'd&nbsp;+1 this, if the reaping logic weren't already causing me headaches.</li><li>The streams are now instances of UNIXProcess.ProcessPipeOutputStream and ProcessPipeInputStream. Don't get excited...they're still just buffered wrappers around File streams.</li><li>The logic run when the child process exist has changed...with catastrophic consequences.</li></ul><div>Here's the new stream setup and reaper logic:</div></div><div><br /><script class="brush: java;" type="syntaxhighlighter"><![CDATA[ void initStreams(int[] fds) throws IOException { stdin = (fds[0] == -1) ? ProcessBuilder.NullOutputStream.INSTANCE : new ProcessPipeOutputStream(fds[0]); stdout = (fds[1] == -1) ? ProcessBuilder.NullInputStream.INSTANCE : new ProcessPipeInputStream(fds[1]); stderr = (fds[2] == -1) ? ProcessBuilder.NullInputStream.INSTANCE : new ProcessPipeInputStream(fds[2]); processReaperExecutor.execute(new Runnable() { public void run() { int exitcode = waitForProcessExit(pid); UNIXProcess.this.processExited(exitcode); }}); } ]]></script></div><div><br /></div><div>Now instead of simply notifying the UNIXProcess that the child has died, there's a call to processExited().</div><div><br /></div><div><script class="brush: java;" type="syntaxhighlighter"><![CDATA[ void processExited(int exitcode) { synchronized (this) { this.exitcode = exitcode; hasExited = true; notifyAll(); } if (stdout instanceof ProcessPipeInputStream) ((ProcessPipeInputStream) stdout).processExited(); if (stderr instanceof ProcessPipeInputStream) ((ProcessPipeInputStream) stderr).processExited(); if (stdin instanceof ProcessPipeOutputStream) ((ProcessPipeOutputStream) stdin).processExited(); } ]]></script></div><div><br /></div><div>Ok, doesn't look bad so far. Let's look at ProcessPipeInputStream, which handles output from the child process.</div><div><br /></div><div><script class="brush: java;" type="syntaxhighlighter"><![CDATA[ /** * A buffered input stream for a subprocess pipe file descriptor * that allows the underlying file descriptor to be reclaimed when * the process exits, via the processExited hook. * * This is tricky because we do not want the user-level InputStream to be * closed until the user invokes close(), and we need to continue to be * able to read any buffered data lingering in the OS pipe buffer. */ static class ProcessPipeInputStream extends BufferedInputStream { ProcessPipeInputStream(int fd) { super(new FileInputStream(newFileDescriptor(fd))); } private static byte[] drainInputStream(InputStream in) throws IOException { if (in == null) return null; int n = 0; int j; byte[] a = null; while ((j = in.available()) > 0) { a = (a == null) ? new byte[j] : Arrays.copyOf(a, n + j); n += in.read(a, n, j); } return (a == null || n == a.length) ? a : Arrays.copyOf(a, n); } /** Called by the process reaper thread when the process exits. */ synchronized void processExited() { // Most BufferedInputStream methods are synchronized, but close() // is not, and so we have to handle concurrent racing close(). try { InputStream in = this.in; if (in != null) { byte[] stragglers = drainInputStream(in); in.close(); this.in = (stragglers == null) ? ProcessBuilder.NullInputStream.INSTANCE : new ByteArrayInputStream(stragglers); if (buf == null) // asynchronous close()? this.in = null; } } catch (IOException ignored) { // probably an asynchronous close(). } } } ]]></script></div><div><br /></div><div>So when the child process exits, the any data waiting to be read from its output stream is drained into a buffer. <b>All of it. In memory.</b></div><div><br /></div><div>Did you launch a process that writes a gigabyte of data to its output stream and then terminates? Well, friend, I sure hope you have a gigabyte of memory, because the JDK is going to read that sucker in and there's nothing you can do about it. And let's hope there's not more than 2GB of data, since this code basically just grows a byte[], which in Java can only grow to 2GB. If there's more than 2GB of data on that stream, this logic errors out and the data is lost forever. Oh, and by the way...if you happened to be devlishly clever and managed to dig down to the real FileChannel attached to the child process, all the data from that stream has suddenly disappeared, and the channel itself is closed, even if you never got a chance to read from it. Thanks for the help, JDK.</div><div><br /></div><div>The JDK has managed to both break our clever workarounds (for its previously broken logic) an break itself even more badly. It's almost like they want to make subprocess launching so dreadfully bad you just don't use it anymore.</div><div><br /></div><h4 style="text-align: left;">Never Surrender</h4><div><br /></div><div>Of course I could cry into my beer over this, but these sorts of problems and challenges are exactly why I'm involved in JRuby and OpenJDK. Obviously this API has gone off the deep end and can't be saved, so what's a hacker to do? In our case, we make our own API.</div><div><br /></div><div>At this point, that's our only option. The ProcessBuilder and Process APIs are so terribly broken that we can't rely on them anymore. Thankfully, JRuby ships with a solid, fast FFI layer called the Java Native Runtime (JNR) that should make it possible for us to write our own process API entirely in Java. We will of course do that in the open, and we are hoping you will help us.</div><div><br /></div><div>What's the moral of the story? I don't really know. Perhaps it's that lowest-common-denominator APIs usually trend toward uselessness. Perhaps it's that ignoring POSIX is an expressway to failure. Perhaps it's that I don't know when to quit. In any case, you can count on the JRuby team to continue bringing you the only true POSIX experience on the JVM, and you can count on me to keep pushing OpenJDK to follow our lead.</div></div></div></div>
|
|
73
|
+
</content>
|
|
74
|
+
<link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/462657466694269626/comments/default' title='Post Comments'/>
|
|
75
|
+
<link rel='replies' type='text/html' href='http://blog.headius.com/2013/06/the-pain-of-broken-subprocess.html#comment-form' title='12 Comments'/>
|
|
76
|
+
<link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/462657466694269626'/>
|
|
77
|
+
<link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/462657466694269626'/>
|
|
78
|
+
<link rel='alternate' type='text/html' href='http://blog.headius.com/2013/06/the-pain-of-broken-subprocess.html' title='The Pain of Broken Subprocess Management on JDK'/>
|
|
79
|
+
<author>
|
|
80
|
+
<name>Charles Nutter</name>
|
|
81
|
+
<uri>https://plus.google.com/101599370339210456684</uri>
|
|
82
|
+
<email>noreply@blogger.com</email>
|
|
83
|
+
<gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/>
|
|
84
|
+
</author>
|
|
85
|
+
<thr:total>12</thr:total>
|
|
86
|
+
</entry>
|
|
87
|
+
<entry>
|
|
88
|
+
<id>tag:blogger.com,1999:blog-4704664917418794835.post-681101033932402497</id>
|
|
89
|
+
<published>2013-05-11T03:05:00.001-07:00</published>
|
|
90
|
+
<updated>2013-05-11T03:10:32.549-07:00</updated>
|
|
91
|
+
<title type='text'>On Languages, VMs, Optimization, and the Way of the World</title>
|
|
92
|
+
<content type='html'><div dir="ltr" style="text-align: left;" trbidi="on"><div style="color: #222222; font-family: arial; font-size: small;">I shouldn't be up this late, but I've been doing lots of thinking and exploring tonight.</div><div style="color: #222222; font-family: arial; font-size: small;"><br />In studying various VMs over the past few years, I've come up with a list of do's and don't that make things optimize right. These apply to languages, the structures that back them, and the VMs that optimize those languages, and from what I've seen there's a lot of immutable truths here given current optimization technology.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">Let's dive in.<br /><br /></div><h3 style="text-align: left;">#1: Types don't have to be static</h3><div style="color: #222222; font-family: arial; font-size: small;"><br />JVM and other dynamic-optimizing runtimes have proven this out. At runtime, it's possible to gather the same information static types would provide you at compile time, leading to optimizations at least as good as fully statically-typed, statically-optimized code. In some cases, it may be possible to do a better job, since runtime profiling is based on real execution, real branch percentages, real behavior, rather than a guess at what a program might do. You could probably make the claim that static optimization is a halting problem, and dynamic optimization eventually can beat it by definition since it can optimize what the program is actually doing.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">However, this requires one key thing to really work well.<br /><br /></div><h3 style="text-align: left;">#2: Types need to be predictable</h3><div style="color: #222222; font-family: arial; font-size: small;"><br />In order for runtime optimization to happen, objects need to have predictable types and those types need to have a predictable structure. This isn't to say that types must be statically declared...they just need to look the same on repeat visits. If objects can change type (smalltalk's become, perl's and C's weak typing) you're forced to include more guards against those changes, or you're forced to invalidate more code whenever something changes (or in the case of C, you just completely shit the bed when things aren't as expected). If change is possible and exposed at a language level, there may be nothing you can do to cope with all those different type shapes, and optimization can only go so far.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">This applies both to the shape of a type's method table (methods remaining consistent once encountered) and the shape of the type's instances (predictable object layout). Many dynamically-typed languages impose dynamic type shape and object shape on VMs that run them, preventing those VMs from making useful predictions about how to optimize code. Optimistic predictions (generating synthetic types for known type shapes or preemptively allocating objects based on previously-seen shapes) still have to include fallback logic to maintain the mutable behavior, should it ever be needed. Again, optimization potential is limited, because the shape of the world can change on a whim and the VM has to be vigilent</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">The alternative summation of #1 and #2 is that types don't have to be statically declared, but they need to be statically defined. Most popular dynamic languages do neither, but all they really need to do is the latter.<br /><br /></div><h3 style="text-align: left;">#3: You can't cheat the CPU</h3><div style="color: #222222; font-family: arial; font-size: small;"><br />Regardless of how clever you'd like to be in your code or language or VM or JIT, the limiting factor is how modern CPUs actually run your code. There's a long list of expectations you must meet to squeeze every last drop of speed out of a system, and diverging from those guidelines will always impose a penalty. This is the end...the bottom turtle...the unifying theory. It is, at the end of the day, the CPU you must appease to get the best performance. All other considerations fall out of that, and anywhere performance does not live up to expectations you are guaranteed to discover that someone tried to cheat the CPU.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">Traditionally, static typing was the best way to guarantee we produced good CPU instructions. It gave us a clear picture of the world we could ponder and meditate over, eventually boiling out the secrets of the universe and producing the fastest possible code. But that always assumed a narrow vision of a world with unlimited resources. It assumed we could make all the right decisions for a program ahead of time and that no limitations outside our target instruction set would ever affect us. In the real world, however, CPUs have limited cache sizes, multiple threads, bottlenecked memory pipelines, and basic physics to contend with (you can only push so many electrons through a given piece of matter without blowing it up). Language and VM authors ignore the expectations of their target systems only at great peril.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">Let's look at a few languages and where they fit.<br /><br /></div><h3 style="text-align: left;">Language Scorecard</h3><div style="color: #222222; font-family: arial; font-size: small;"><br />Java is statically typed and types are of a fixed shape. This is the ideal situation mostly because of the type structure being predictable. Once encountered, a rose is just a rose. Given appropriate dynamic optimizations, there's no reason Java code can't compete with or surpass statically-typed and statically-compiled C/++, and in theory there's nothing preventing Java code from becoming optimal CPU instructions.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">Dart is dynamically typed (or at least, types are optional and the VM doesn't care about them), but types are of a fixed shape. If programmers can tolerate fixed-shape types, Dart provides a very nice dynamic language that still can achieve the same optimizations as statically-typed Java or statically-compiled C/++.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">Groovy is dynamically typed with some inference and optimization if you specify static types, but most (all?) types defined in Groovy are not guaranteed to be a fixed shape. As a result, even when specifying static types, guards must be inserted to check that those types' shapes have not changed. Groovy does, however, guarantee object shape is consistent over time, which avoids overhead from being able to reshape objects at runtime.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">Ruby and JavaScript are dynamically typed and types and objects can change shape at runtime. This is a confluence of all the hardest-to-optimize language characteristics. In both cases, the best we can do is to attempt to predict common type and object shapes and insert guards for when we're wrong, but it's not possible to achieve the performance of a system with fully-predictable type and object shapes. Prove me wrong.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">Now of course when I say it's not possible, I mean it's not possible for the general case. Specific cases of a known closed-world application can indeed be optimized as though the types and objects involved had static shapes. I do something along these lines in my RubyFlux compiler, which statically analyzes incoming Ruby code and assumes the methods it sees defined and the fields it sees accessed will be the only methods and fields it ever needs to worry about. But that requires omitting features that can mutate type and object structure, or else you have to have a way to know which types and objects those features will affect. Sufficiently smart compiler indeed.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">Python has similar structural complexities to Ruby and adds in the additional complexity of an introspectable call stack. Under those circumstances, even on-stack execution state is not safe; a VM can't even make guarantees about the values it has in hand or the shape of a given call's activation. PyPy does an admirable job of attacking this problem by rewriting currently-running code and lifting on-stack state to the heap when it is accessed, but this approach prevents dropping unused local state (since you can't predict who might want to see it) and also fails to work under parallel execution (since you can't rewrite code another thread might be executing). Again, the dynamicity of a "cool" feature brings with it intrinsic penalties that are reducible but not removable.<br /><br /></div><h3 style="text-align: left;">Get to the Damn Point, Already</h3><div style="color: #222222; font-family: arial; font-size: small;"><br />So what am I trying to say in all this? I started the evening by exploring a benchmark post comparing Dart's VM with JVM on the same benchmark. The numbers were not actually very exciting...with a line-by-line port from Dart to Java, Java came out slightly behind Dart. With a few modifications to the Java code, Java pulled slightly ahead. With additional modifications to the Dart code, it might leapfrog Java again. But this isn't interesting because Dart and Java can both rely on type and object shapes remaining consistent, and as a result the optimizations they perform can basically accomplish the same thing. Where it matters, they're similar enough that VMs don't care about the differences.</div><div style="color: #222222; font-family: arial; font-size: small;"><br /></div><div style="color: #222222; font-family: arial; font-size: small;">Where does this put languages I love, like Ruby? It's probably fair to concede that Ruby can't ever achieve the raw, straight-line performance of type-static (not statically-typed) languages like Dart or Java, regardless of the VM technologies involved. We'll be able to get close; JRuby can, with the help of invokedynamic, make method calls *nearly* as fast as Java calls, and by generating type shapes we can make object state *nearly* as predictable as Java types, but we can't go all the way. Regardless of how great the underlying VM is, if you can't hold to its immutable truths, you're walking against the wind. Ruby on Dart would probably not be any faster than Ruby on JVM, because you'd still have to implement mutable types and growable objects in pretty much the same way. Ruby on PyPy might be able to go farther, since the VM is designed for mutable types and growable objects, but you might have to sacrifice parallelism or accept that straight-line object-manipulating performance won't go all the way to a Java or Dart. Conversely, languages that make those type-static guarantees might be able to beat dynamic languages when running on dynamic language VMs (e.g. dart2js) for exactly the same reasons that they excel on their own VMs: they provide a more consistent view of the world, and offer no surprises to the VM that would hinder optimization. You trade dynamicity at the language level for predictability at the VM level.<br /><br /></div><h3 style="text-align: left;">The Actual Lesson</h3><div style="color: #222222; font-family: arial; font-size: small;"><br />I guess the bottom line for me is realizing that there's always going to be a conflict between what programmers want out of programming languages and what's actually possible to give them. There's no magical fairy world where every language can be as fast as every other language, because there's no way to predict how every program is going to execute (or in truth, how a given program is going to execute given a general strategy). And that's ok; most of these languages can still get very close to each other in performance, and over time the dynamic type/object-shaped languages may offer ways to ratchet down some of that dynamism...or they might not care and just accept what limitations result. The important thing is for language users to recognize that nothing is free, and to understand the implications of language features and design decisions they make in their own programs.</div></div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/681101033932402497/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2013/05/on-languages-vms-optimization-and-way.html#comment-form' title='15 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/681101033932402497'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/681101033932402497'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2013/05/on-languages-vms-optimization-and-way.html' title='On Languages, VMs, Optimization, and the Way of the World'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>15</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-5392245422382118146</id><published>2013-01-05T08:47:00.000-08:00</published><updated>2013-01-05T08:47:32.452-08:00</updated><category scheme="http://www.blogger.com/atom/ns#" term="invokedynamic"/><category scheme="http://www.blogger.com/atom/ns#" term="jruby"/><category scheme="http://www.blogger.com/atom/ns#" term="jvm"/><category scheme="http://www.blogger.com/atom/ns#" term="optimization"/><category scheme="http://www.blogger.com/atom/ns#" term="ruby"/><title type='text'>Constant and Global Optimization in JRuby 1.7.1 and 1.7.2</title><content type='html'><div dir="ltr" style="text-align: left;" trbidi="on">With every JRuby release, there's always at least a handful of optimizations. They range from tiny improvements in the compiler to perf-aware rewrites of core class methods, but they're almost always driven by real-world cases.<br /><br />In JRuby 1.7.1 and 1.7.2, I made several improvements to the performance of Ruby constants and global variables that might be of some interest to you, dear reader.<br /><br /><h2 style="text-align: left;">Constants</h2><div>In Ruby, a constant is a lexically and hierarchically accessed variable that starts with a capital letter. Class and module names like Object, Kernel, String, are all constants defined under the Object class. When I say constants are both lexical and hierarchically accessed, what I mean is that at access time we first search outward through lexically-enclosing scopes, and failing that we search through the class hierarchy of the innermost scope. For example:</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file1.rb"></script></div><div><br /></div><div>Here, the first two constant accesses inside class B are successful; the first (IN_FOO) is located lexically in Foo, because it encloses the body of class B. The second (IN_A) is located hierarchically by searching B's ancestors. The third access fails, because the IN_BAR constant is only available within the Bar module's scope, so B can't see it.</div><div><br /></div><div>Constants also...aren't. It is possible to redefine a constant, or define new constants deeper in a lexical or hierarchical strcture that mask earlier ones. However in most code (i.e. "good" code) constants eventually stabilize. This makes it possible to perform a variety of optimizations against them, even though they're not necessarily static.</div><div><br /></div><div>Constants are used heavily throughout Ruby, both for constant values like Float::MAX and for classes like Array or Hash. It is therefore especially important that they be as fast as possible.<br /><br /></div><h2 style="text-align: left;">Global Variables</h2><div>Globals in Ruby are about like you'd expect...name/value pairs in a global namespace. They start with &nbsp;$ character. Several global variables are "special" and exist in a more localized source, like $~ (last regular expression match in this call frame), $! (last exception raised in this thread), and so on. Use of these "local globals" mostly just amounts to special variable names that are always available; they're not really true global variables.</div><div><br /></div><div>Everyone knows global variables should be discouraged, but that's largely referring to global variable use in normal program flow. Using global state across your application – potentially across threads – is a pretty nasty thing to do to yourself and your coworkers. But there are some valid uses of globals, like for logging state and levels, debugging flags, and truly global constructs like standard IO.</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file2.rb"></script></div><div><br /></div><div>Here, we're using the global $DEBUG to specify whether logging should occur in MyApp#log. Those log messages are written to the stderr stream accessed via $stderr. Note also that $DEBUG can be set to true by passing -d at the JRuby command line.<br /><br /></div><h2 style="text-align: left;">Optimizing Constant Access (pre-1.7.1)</h2><div>I've posted in the past about how JRuby optimizes constant access, so I'll just quickly review that here.</div><div><br /></div><div>At a given access point, constant values are looked up from the current lexical scope and cached. Because constants can be modified, or new constants can be introduce that mask earlier ones, the JRuby runtime (org.jruby.Ruby) holds a global constant invalidator checked on each access to ensure the previous value is still valid.</div><div><br /></div><div>On non-invokedynamic JVMs, verifying the cache involves an object identity comparison every time, which means a non-final value must be accessed via a couple levels of indirection. This adds a certain amount of overhead to constant access, and also makes it impossible for the JVM to fold multiple constant accesses away, or make static decisions based on a constant's value.</div><div><br /></div><div>On an invokedynamic JVM, the cache verification is in the form of a SwitchPoint. SwitchPoint is a type of on/off guard used at invokedynamic call sites to represent a hard failure. Because it can only be switched off, the JVM is able to optimize the SwitchPoint logic down to what's called a "safe point", a very inexpensive ping back into the VM. As a result, constant accesses under invokedynamic can be folded away, and repeat access or unused accesses are not made at all.</div><div><br /></div><div>However, there's a problem. In JRuby 1.7.0 and earlier, the only way we could access the current lexical scope (in a StaticScope object) was via the current call frame's DynamicScope, a heap-based object created on each activation of a given body of code. In order to reduce the performance hit to methods containing constants, we introduced a one-time DynamicScope called the "dummy scope", attached to the lexical scope and only created once. This avoided the huge hit of constructing a DynamicScope for every call, but caused constant-containing methods to be considerably slower than those without constants.<br /><br /></div><h2 style="text-align: left;">Lifting Lexical Scope Into Code</h2><div>In JRuby 1.7.1, I decided to finally bite the bullet and make the lexical scope available to all method bodies, without requiring a DynamicScope intermediate. This was a&nbsp;<a href="https://github.com/jruby/jruby/compare/fb65c539a9b4f52d1d063dbe36de69217ab6a896...ad5d07291d09f57849f873d405607fbb6fed1544">nontrivial piece of work</a>&nbsp;that took several days to get right, so although most of the work occurred before JRuby 1.7.0 was released, we opted to let it bake a bit before release.</div><div><br /></div><div>The changes made it possible for all class, module, method, and block bodies to access their lexical scope essentially for free. It also helped us finally deliver on the promise of truly free constant access when running under invokedynamic.</div><div><br /></div><div>So, does it work?</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file3.rb"></script></div><div><br /></div><div>Assuming constant access is free, the three loops here should perform identically. The non-expression calls to foo and bar should disappear, since they both return a constant value that's never used. The calls for decrementing the 'a' variable should produce a constant value '1' and perform the same as the literal decrement in the control loop.</div><div><br /></div><div>Here's Ruby (MRI) 2.0.0 performance on this benchmark.</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file4.rb"></script></div><div><br /></div><div>The method call itself adds a significant amount of overhead here, and the constant access adds another 50% of that overhead. Ruby 2.0.0 has done a lot of work on performance, but the cost of invoking Ruby methods and accessing constants remains high, and constant accesses do not fold away as you would like.</div><div><br /></div><div>Here's JRuby 1.7.2 performance on the same benchmark.</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file5.rb"></script></div><div><br /></div><div>We obviously run all cases significantly faster than Ruby 2.0.0, but the important detail is that the method call adds only about 11% overhead to the control case, and constant access adds almost nothing.</div><div><br /></div><div>For comparison, here's JRuby 1.7.0, which did not have free access to lexical scopes.</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file6.rb"></script></div><div><br /></div><div>So by avoiding the intermediate DynamicScope, methods containing constant accesses are somewhere around 7x faster than before. Not bad.<br /><br /></div><h2 style="text-align: left;">Optimizing Global Variables</h2><div>Because global variables have a much simpler structure than constants, they're pretty easy to optimize. I had not done so up to JRuby 1.7.1 mostly because I didn't see a compelling use case and didn't want to encourage their use. However, after Tony Arcieri pointed out that invokedynamic-optimized global variables could be used to add logging and profiling to an application with zero impact when disabled, I was convinced. Let's look at the example from above again.</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file2.rb"></script></div><div><br /></div><div>In this example, we would ideally like there to be no overhead at all when $DEBUG is untrue, so we're free to add optional logging throughout the application with no penalty. In order to support this, two improvements were needed.</div><div><br /></div><div>First, I modified our invokedynamic logic to cache global variables using a per-variable SwitchPoint. This makes access to mostly-static global variables as free as constant access, with the same performance improvements.</div><div><br /></div><div>Second, I added some smarts into the compiler for conditional forms like "if $DEBUG" that would avoid re-checking the $DEBUG value at all if it were false the first time (and start checking it again if it were modified).</div><div><br /></div><div>It's worth noting I also made this second optimization for constants; code like "if DEBUG_ENABLED" will also have the same performance characteristics.</div><div><br /></div><div>Let's see how it performs.</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file7.rb"></script></div><div><br /></div><div>In this case, we should again expect that all three forms have identical performance. Both the constant and the global resolve to an untrue value, so they should ideally not introduce any overhead compared to the bare method.</div><div><br /></div><div>Here's Ruby (MRI) 2.0.0:</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file8.rb"></script></div><div><br /></div><div>Both the global and the constant add overhead here in the neighborhood of 25% over an empty method. This means you can't freely add globally-conditional logic to your application without accepting a performance hit.</div><div><br /></div><div>JRuby 1.7.2:</div><div><br /></div><div><script src="https://gist.github.com/4459891.js?file=file9.rb"></script></div><div><br /></div><div>Again we see JRuby +&nbsp;invokedynamic optimizing method calls considerably better than MRI, but additionally we see that the untrue global conditions add no overhead compared to the empty method. You can freely use globals as conditions for logging, profiling, and other code you'd like to have disabled most of the time.</div><div><br /></div><div>And finally, JRuby 1.7.1, which optimized constants, did not optimize globals, and did not have specialized conditional logic for either:<br /><br /></div><div><script src="https://gist.github.com/4459891.js?file=file10.rb"></script></div><h2 style="text-align: left;">Where Do We Go From Here?</h2><div>Hopefully I've helped show that we're really just seeing the tip of the iceberg as far as optimizing JRuby using invokedynamic. More than anything we want you to report real-world use cases that could benefit from additional optimization, so we can target our work effectively. And as always, please try out your apps on JRuby, enable JRuby testing in Travis CI, and let us know what we can do to make your JRuby experience better!</div></div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/5392245422382118146/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2013/01/constant-and-global-optimization-in.html#comment-form' title='4 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5392245422382118146'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5392245422382118146'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2013/01/constant-and-global-optimization-in.html' title='Constant and Global Optimization in JRuby 1.7.1 and 1.7.2'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>4</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-5309576998658669333</id><published>2012-11-19T06:38:00.001-08:00</published><updated>2012-11-19T10:36:04.211-08:00</updated><title type='text'>Refining Ruby</title><content type='html'><div dir="ltr" style="text-align: left;" trbidi="on">What does the following code do?<br /><br /><script src="https://gist.github.com/4110634.js?file=ref_1.rb"></script> If you answered "it upcases two strings and adds them together, returning the result" you might be wrong because of a new Ruby feature called "refinements".<br /><br />Let's start with the problem refinements are supposed to solve: monkey-patching.<br /><br /><h2 style="text-align: left;">Monkey-patching</h2><div>In Ruby, all classes are mutable. Indeed, when you define a new class, you're really just creating an empty class and filling it with methods. The ability to mutate classes at runtime has been used (or abused) by many libraries and frameworks to decorate Ruby's core classes with additional (or replacement) behavior. For example, you might add a "camelize" method to String that knows how to convert under_score_names to camelCaseNames. This is lovingly called "monkey-patching" by the Ruby community.</div><div><br /></div><div>Monkey-patching can be very useful, and many patterns in Ruby are built around the ability to modify classes. It can also cause problems if a library patches code in a way the user does not expect (or want), or if two libraries try to apply conflicting patches. Sometimes, you simply don't want patches to apply globally, and this is where refinements come in.<br /><br /></div><h2 style="text-align: left;">Localizing Monkeypatches</h2><div>Refinements have been discussed as a feature for several years, sometimes under the name "selector namespaces". In essence, refinements are intended to allow monkey-patching only within certain limited scopes, like within a library that wants to use altered or enhanced versions of core Ruby types without affecting code outside the library. This is the case within the ActiveSupport library that forms part of the core of Rails.</div><div><br /></div><div>ActiveSupport provides a number of extensions (patches) to the core Ruby classes like String#pluralize, Range#overlaps?, and Array#second. Some of these extensions are intended for use by Ruby developers, as conveniences that improve the readability or conciseness of code. Others exist mostly to support Rails itself. In both cases, it would be nice if we could prevent those extensions from leaking out of ActiveSupport into code that does not want or need them.<br /><br /></div><h2 style="text-align: left;">Refinements</h2><div>In short, refinements provide a way to make class modifications that are only seen from within certain scopes. In the following example, I add a "camelize" method to the String class that's only seen from code within the Foo class.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_2.rb"></script> <br /><div><br /></div><div>With the Foo class refined, we can see that the "camelize" method is indeed available within the "camelize_string" method but not outside of the Foo class.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_3.txt"></script> <br /><div><br /></div><div>On the surface, this seems like exactly what we want. Unfortunately, there's a lot more complexity here than meets the eye.<br /><br /></div><h2 style="text-align: left;">Ruby Method Dispatch</h2><div>In order to do a method call in Ruby, a runtime simply looks at the target object's class hierarchy, searches for the method from bottom to top, and upon finding it performs the call. A smart runtime will cache the method to avoid performing this search every time, but in general the mechanics of looking up a method body are rather simple.</div><div><br /></div><div>In an implementation like JRuby, we might cache the method at what's called the "call site"—the point in Ruby code where a method call is actually performed. In order to know that the method is valid for future calls, we perform two checks at the call site: that the incoming object is of the same type as for previous calls; and that the type's hierarchy has not been mutated since the method was cached.</div><div><br /></div><div>Up to now, method dispatch in Ruby has depended solely on the type of the target object. The calling context has not been important to the method lookup process, other than to confirm that visibility restrictions are enforced (primarily for protected methods, since private methods are rejected for non–self calls). That simplicity has allowed Ruby implementations to optimize method calls and Ruby programmers to understand code by simply determining the target object and methods available on it.</div><div><br /></div><div>Refinements change everything.<br /><br /></div><h2 style="text-align: left;">Refinements Basics</h2><div>Let's revisit the camelize example again.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_2.rb"></script> <br /><div><br /></div><div>The visible manifestation of refinements comes via the "refine" and "using" methods.</div><div><br /></div><div>The "refine" method takes a class or module (the String class, in this case) and a block. Within the block, methods defined (camelize) are added to what might be called a patch set (a la monkey-patching) that can be applied to specific scopes in the future. The methods are not actually added to the refined class (String) except in a "virtual" sense when a body of code activates the refinement via the "using" method.</div><div><br /></div><div>The "using" method takes a refinement-containing module and applies it to the current scope. Methods within that scope should see the refined version of the class, while methods outside that scope do not.</div><div><br /></div><div>Where things get a little weird is in defining exactly what that scope should be and in implementing refined method lookup in such a way that does not negatively impact the performance of unrefined method lookup. In the current implementation of refinements, a "using" call affects all of the following scopes related to where it is called:</div><div><ul style="text-align: left;"><li>The direct scope, such as the top-level of a script, the body of a class, or the body of a method or block</li><li>Classes down-hierarchy from a refined class or module body</li><li>Bodies of code run via eval forms that change the "self" of the code, such as module_eval</li></ul><div>It's worth emphasizing at this point that refinements can affect code far away from the original "using" call site. It goes without saying that refined method calls must now be aware of both the target type and the calling scope, but what of unrefined calls?<br /><br /></div></div><h2 style="text-align: left;">Dynamic Scoping of Method Lookup</h2><div>Refinements (in their current form) basically cause method lookup to be dynamically scoped. In order to properly do a refined call, we need to know what refinements are active for the context in which the call is occurring and the type of the object we're calling against. The latter is simple, obviously, but determining the former turns out to be rather tricky.<br /><br /></div><h3 style="text-align: left;">Locally-applied refinements</h3><div>In the simple case, where a "using" call appears alongside the methods we want to affect, the immediate calling scope contains everything we need. Calls in that scope (or in child scopes like method bodies) would perform method lookup based on the target class, a method name, and the hierarchy of scopes that surrounds them. The key for method lookup expands from a simple name to a name plus a call context.<br /><br /></div><h3 style="text-align: left;">Hierarchically-applied refinements</h3><div>Refinements applied to a class must also affect subclasses, so even when we don't have a "using" call present we still may need to do refined dispatch. The following example illustrates this with a subclass of Foo (building off the previous example).</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_4.rb"></script> <br /><div><br /></div><div>Here, the camelize method is used within a "map" call, showing that refinements used by the Foo class apply to Bar, its method definitions, and any subscopes like blocks within those methods. It should be apparent now why my first example might not do what you expect. Here's my first example again, this time with the Quux class visible.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_5.rb"></script> <br /><div><br /></div><div>The Quux class uses refinements from the BadRefinement module, effectively changing String#upcase to actually do String#reverse. By looking at the Baz class alone you can't tell what's supposed to happen, even if you are certain that str1 and str2 are always going to be String. Refinements have effectively localized the changes applied by the BadRefinement module, but they've also made the code more difficult to understand; the programmer (or the reader of the code) must know everything about the calling hierarchy to reason about method calls and expected results.<br /><br /></div><h3 style="text-align: left;">Dynamically-applied refinements</h3><div>One of the key features of refinements is to allow block-based DSLs (domain-specific languages) to decorate various types of objects without affecting code outside the DSL. For example, an RSpec spec.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_6.rb"></script> <br /><div><br /></div><div>There's several calls here that we'd like to refine.</div><div><ul style="text-align: left;"><li>The "describe" method is called at the top of the script against the "toplevel" object (essentially a singleton Object instance). We'd like to apply a refinement at this level so "describe" does not have to be defined on Object itself.</li><li>The "it" method is called within the block passed to "describe". We'd like whatever self object is live inside that block to have an "it" method without modifying self's type directly.</li><li>The "should" method is called against an instance of MyClass, presumably a user-created class that does not define such a method. We would like to refine MyClass to have the "should" method only within the context of the block we pass to "it".</li><li>Finally, the "be_awesome" method—which RSpec translates into a call to MyClass#awesome?—should be available on the self object active in the "it" block without actually adding be_awesome to self's type.</li></ul><div>In order to do this without having a "using" present in the spec file itself, we need to be able to dynamically apply refinements to code that might otherwise not be refined. The current implementation does this via Module#module_eval (or its argument-receiving brother, Module#module_exec).</div></div><div><br /></div><div>A block of code passed to "module_eval" or "instance_eval" will see its self object changed from that of the original surrounding scope (the self at block creation time) to the target class or module. This is frequently used in Ruby to run a block of code as if it were within the body of the target class, so that method definitions affect the "module_eval" target rather than the code surrounding the block.</div><div><br /></div><div>We can leverage this behavior to apply refinements to any block of code in the system. Because refined calls must look at the hierarchy of classes in the surrounding scope, every call in every block in every piece of code can potentially become refined in the future, if the block is passed via module_eval to a refined hierarchy. The following simple case might not do what you expect, even if the String class has not been modified directly.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_7.rb"></script> <br /><div><br /></div><div>Because the "+" method is called within a block, all bets are off. The str_ary passed in might not be a simple Array; it could be any user class that implements the "inject" method. If that implementation chooses, it can force the incoming block of code to be refined. Here's a longer version with such an implementation visible.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_8.rb"></script> <br /><div><br /></div><div>Suddenly, what looks like a simple addition of two strings produces a distinctly different result.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_9.txt"></script> <br /><div><br /></div><div>Now that you know how refinements work, let's discuss the problems they create.<br /><br /></div><h2 style="text-align: left;">Implementation Challenges</h2><div>Because I know that most users don't care if a new, useful feature makes my life as a Ruby implementer harder, I'm not going to spend a great deal of time here.&nbsp;My concerns revolve around the complexities of knowing when to do a refined call and how to discover those refinements.</div><div><br /></div><div>Current Ruby implementations are all built around method dispatch depending solely on the target object's type, and much of the caching and optimization we do depends on that. With refinements in play, we must also search and guard against types in the caller's context, which makes lookup much more complicated. Ideally we'd be able to limit this complexity to only refined calls, but because "using" can affect code far away from where it is called, we often have no way to know whether a given call might be refined in the future. This is especially pronounced in the "module_eval" case, where code that isn't even in the same class hierarchy as a refinement must still observe it.</div><div><br /></div><div>There are numerous ways to address the implementation challenges.<br /><br /></div><h3 style="text-align: left;">Eliminate the "module_eval" Feature</h3><div>At present, nobody knows of an easy way to implement the "module_eval" aspect of refinements. The current implementation in MRI does it in a brute-force way, flushing the global method cache on every execution and generating a new, refined, anonymous module for every call. Obviously this is not a feasible direction to go; block dispatch will happen very frequently at runtime, and we can't allow refined blocks to destroy performance for code elsewhere in the system.</div><div><br /></div><div>The basic problem here is that in order for "module_eval" to work, every block in the system must be treated as a refined body of code all the time. That means that calls inside blocks throughout the system need to search and guard against the calling context even if no refinements are ever applied to them. The end result is that those calls suffer complexity and performance hits across the board.</div><div><br /></div><div>At the moment, I do not see (nor does anyone else see) an efficient way to handle the "module_eval" case. It should be removed.<br /><br /></div><h3 style="text-align: left;">Localize the "using" Call</h3><div>No new Ruby feature should cause across-the-board performance hits; one solution is for refinements to be recognized at parse time. This makes it easy to keep existing calls the way they are and only impose refinement complexity upon method calls that are actually refined.</div><div><br /></div><div>The simplest way to do this is also the most limiting and the most cumbersome: force "using" to only apply to the immediate scope. This would require every body of code to "using" a refinement if method calls in that body should be refined. Here's a couple of our previous examples with this modification.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref10.rb"></script> <br /><div><br /></div><div>This is obviously pretty ugly, but it makes implementation much simpler. In every scope where we see a "using" call, we simply force all future calls to honor refinements. Calls appearing outside "using" scopes do not get refined and perform calls as normal.</div><div><br /></div><div>We can improve this by making "using" apply to child scopes as well. This still provides the same parse-time "pseudo-keyword" benefit without the repetition.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref11.rb"></script> <br /><div><br /></div><div>Even better would be to officially make "using" a keyword and have it open a refined scope; that results in a clear delineation between refined and unrefined code. I show two forms of this below; the first opens a scope like "class" or "module", and the second uses a "do...end" block form.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref12.rb"></script> <br /><div><br /></div><div>It would be fair to say that requiring more explicit scoping of "using" would address my concern about knowing when to do a refined call. It does not, however, address the issues of locating active refinements at call time.<br /><br /></div><h3 style="text-align: left;">Locating Refinements</h3><div>In each of the above examples, we still must pass some state from the calling context through to the method dispatch logic. Ideally we'd only need to pass in the calling object, which is already passed through for visibility checking. This works for refined class hierarchies, but it does not work for the RSpec case, since the calling object in some cases is just the top-level Object instance (and remember we don't want to decorate Object).</div><div><br /></div><div>It turns out that there's already a feature in Ruby that follows lexical scoping: constant lookup. When Ruby code accesses a constant, the runtime must first search all enclosing scopes for a definition of that constant. Failing that, the runtime will walk the self object's class hierarchy. This is similar to what we want for the simplified version of refinements.</div><div><br /></div><div>If we assume we've localized refinements to only calls within "using" scopes, then at parse time we can emit something like a RefinedCall for every method call in the code. A RefinedCall would be special in that it uses both the containing scope and the target class to look up a target method. The lookup process would proceed as follows:</div><div><ol style="text-align: left;"><li>Search the call's context for refinements, walking lexical scopes only</li><li>If refinements are found, search for the target method</li><li>If a refined method is found, use it for the call</li><li>Otherwise, proceed with normal lookup against the target object's class</li></ol><div>Because the parser has already isolated refinement logic to specific calls, the only change needed is to pass the caller's context through to method dispatch.<br /><br /></div></div><h2 style="text-align: left;">Usability Concerns</h2><div>There are indeed flavors of refinements that can be implemented reasonably efficiently, or at least implemented in such a way that unrefined code will not pay a price. I believe this is a requirement of any new feature: do no harm. But harm can come in a different form if a new feature makes Ruby code harder to reason about. I have some concerns here.</div><div><br /></div><div>Let's go back to our "module_eval" case.</div><div><br /></div><script src="https://gist.github.com/4110634.js?file=ref_7.rb"></script> <br /><div><br /></div><div>Because there's no "using" anywhere in the code, and we're not extending some other class, most folks will assume we're simply concatenating strings here. After all, why would I expect my "+" call to do something else? Why <b>should</b>&nbsp;my "+" call ever do something else here?</div><div><br /></div><div>Ruby has many features that might be considered a little "magical". In most cases, they're only magic because the programmer doesn't have a good understanding of how they work. Constant lookup, for example, is actually rather simple...but if you don't know it searches both lexical and hierarchical contexts, you may be confused where values are coming from.</div><div><br /></div><div>The "module_eval" behavior of refinements simply goes too far. It forces every Ruby programmer to second-guess every block of code they pass into someone else's library or someone else's method call. The guarantees of standard method dispatch no longer apply; you need to know if the method you're calling will change what calls your code makes. You need to understand the internal details of the target method. That's a terrible, terrible thing to do to Rubyists.</div><div><br /></div><div>The same goes for refinements that are active down a class hierarchy. You can no longer extend a class and know that methods you call actually do what you expect. Instead, you have to know whether your parent classes or their ancestors refine some call you intend to make. I would argue this is considerably <b>worse</b>&nbsp;than directly monkey-patching some class, since at least in that case every piece of code has a uniform view.</div><div><br /></div><div>The problems are compounded over time, too. As libraries you use change, you need to again review them to see if refinements are in play. You need to understand all those refinements just to be able to reason about your own code. And you need to hope and pray two libraries you're using don't define different refinements, causing one half of your application to behave one way and the other half of your application to behave another way.</div><div><br /></div><div>I believe the current implementation of refinements introduces more complexity than it solves, mostly due to the lack of a strict lexical "using". Rubyists should be able to look at a piece of code and know what it does based solely on the types of objects it calls. Refinements make that impossible.<br /><br /><i style="font-weight: bold;">Update:</i>&nbsp;Josh Ballanco points out another usability problem: "using" only affects method bodies defined temporally after it is called. For example, the following code only refines the "bar" method, not the "foo" method.<br /><br /><script src="https://gist.github.com/4110634.js?file=ref13.rb"></script><br />This may simply be an artifact of the current implementation, or it may be specified behavior; it's hard to tell since there's no specification of any kind other than the implementation and a handful of tests. In any case, it's yet another confusing aspect, since it means the order in which code is loaded can actually change which refinements are active.<br /><br /></div><h2 style="text-align: left;">tl;dr</h2><div>My point here is not to beat down refinements. I agree there are cases where they'd be very useful, especially given the sort of monkey-patching I've seen in the wild. But the current implementation overreaches; it provides several features of questionable value, while simultaneously making both performance and understandability harder to achieve. Hopefully we'll be able to work with Matz and ruby-core to come up with a more reasonable, limited version of refinements...or else convince them not to include refinements in Ruby 2.0.</div></div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/5309576998658669333/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2012/11/refining-ruby.html#comment-form' title='19 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5309576998658669333'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5309576998658669333'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2012/11/refining-ruby.html' title='Refining Ruby'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>19</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-6804148115098747648</id><published>2012-10-15T11:26:00.001-07:00</published><updated>2012-10-15T11:40:18.324-07:00</updated><title type='text'>So You Want To Optimize Ruby</title><content type='html'><div dir="ltr" style="text-align: left;" trbidi="on">I was recently asked for a list of "hard problems" a Ruby implementation really needs to solve before reporting benchmark numbers. You know...the sort of problems that might invalidate early perf numbers because they impact how you optimize Ruby. This post is a rework of my response...I hope you find it informative!<br /><h4 style="text-align: left;">Fixnum to Bignum promotion</h4>In Ruby, Fixnum math can promote to Bignum when the result is out of Fixnum's range. On implementations that use tagged pointers to represent Fixnum (MRI, Rubinius, MacRuby), the Fixnum range is somewhat less than the base CPU bits (32/64). On JRuby, Fixnum is always a straight 64-bit signed value.<br /><br />This promotion is a performance concern for a couple reasons:<br /><ul style="text-align: left;"><li>Every math operation that returns a new Fixnum must be range-checked. This slows all Fixnum operations.</li><li>It is difficult (if not impossible) to predict whether a Fixnum math operation will return a Fixnum or a Bignum. Since Bignum is always represented as a full object (not a primitive or a tagged pointer) this impacts optimizing Fixnum math call sites.</li></ul><h4 style="text-align: left;">Floating-point performance</h4>A similar concern is the performance of floating point values. Most of&nbsp;the native implementations have tagged values for Fixnum but only one&nbsp;I know of (Macruby) uses tagged values for Float. This can skew&nbsp;expectations because an implementation may perform very well on integer math and&nbsp;considerably worse on floating-point math due to the objects created (and collected). JRuby uses objects for both Fixnum and Float, so performance is roughly equivalent (and slower than I'd like).<br /><h4 style="text-align: left;">Closures</h4><div>Any language that supports closures ("blocks" in Ruby) has to deal with efficiently accessing frame-local data from calls down-stack. In Java, both anonymous inner classes and the upcoming lambda feature treat frame-local values (local variables, basically) as immutable...so their values can simply be copied into the closure object or carried along in some other way. In Ruby, local variables are always mutable, so an eventual activation of a closure body needs to be able to write into its containing frame. If a runtime does not support arbitrary frame access (as is the case on the JVM) it may have to allocate a separate data structure to represent those frame locals...and that impacts performance.</div><h4 style="text-align: left;">Bindings and eval</h4>The eval methods in Ruby can usually accept an optional binding under which to run. This means any call to binding must return a fully-functional execution environment, and in JRuby this means both eval and binding force a full deoptimization of the surrounding method body.<br /><br />There's an even more unpleasant aspect to this, however: every block can be used as a binding too.<br /><br />All blocks can be&nbsp;turned into Proc and used as bindings, which means every block in the&nbsp;system has to have full access to values in the containing call frame. Most implementers hate this feature, since it means that optimizing call frames in the presence of blocks is much more difficult. Because they can be used as a binding, that of course means&nbsp;literally all frame data must be accessible: local variables;&nbsp;frame-local $ variables like $~; constants lookup environment; method visibility; and so on.<br /><h4 style="text-align: left;">callcc and Continuation</h4>JRuby doesn't implement callcc since the JVM doesn't support continuations, but any implementation hoping to optimize Ruby will have to take a stance here. Continuations obviously make optimization more difficult since you can branch into and out of execution contexts in rather unusual ways.<br /><h4 style="text-align: left;">Fiber implementation</h4>In JRuby, each Fiber runs on its own thread (though we pool the native thread to reduce Fiber spin-up costs). Other than that they&nbsp;operate pretty much like closures.<br /><br />A Ruby implementer needs to decide whether it will use C-style native stack juggling (which makes optimizations like frame elimination trickier to implement) or give Fibers their own stacks in which to execute independently.<br /><h4 style="text-align: left;">Thread/frame/etc local $globals</h4>Thread globals are easy, obviously. All(?) host systems already have some repesentation of thread-local values.&nbsp;The tricky ones are explicit frame&nbsp;globals like $~ and $_ and implicit frame-local values like&nbsp;visibility, etc.<br /><br />In the case of $~ and $_, the challenge is not in representing accesses of them directly but in handling implicit reads and writes of them that cross call boundaries. For example, calling [] on a String and passing a Regexp will cause the caller's frame-local $~ (and related values) to be updated to the MatchData for the pattern match that happens inside [].
|
|
93
|
+
There are a number of core Ruby methods like this that can reach back into the caller's frame and read or write these values. This obviously makes reducing or eliminating call frames very tricky.<br /><br />In JRuby, we track all core methods that read or write these values, and if we see those methods called in a body of code (the names, mind you...this is a static inspection), we will stand up a call frame for that body. This is not ideal. We would like to move these values into a separate stack that's lazily allocated only when actually needed, since methods that cross frames like String#[] force other methods like Array#[] to deoptimize too.<br /><h4 style="text-align: left;">C extension support</h4>If a given Ruby implementation is likely to fit into the "native" side of Ruby&nbsp;implementations (as opposed to implementations like JRuby or IronRuby that target an existing managed runtime), it will need to have a C extension story.<br /><br />Ruby's C&nbsp;extension API is easier to support than some languages' native APIs (e.g. no reference-counting as in Python)&nbsp;but it still very much impacts how a runtime optimizes. Because the API needs to return forever-valid object references, implementations that don't give out pointers will have to maintain a handle table. The API includes a number of macros that provide access to object internals; they'll need to be simulated or explicitly unsupported. And the API makes no guarantees about concurrency and provides few primitives for controlling concurrent execution, so most implementations will need to lock around native downcalls.<br /><br />An alternative for a new Ruby implementation is to expect extensions to be written in the host runtime's native language (Java or other JVM languages for JRuby; C# or other .NET languages for IronRuby, etc). However this imposes a burden on folks implementing language extensions, since they'll have to support yet another language to cover all Ruby implementations.<br /><br />Ultimately, though, the unfortunate fact for most "native" impls is that regardless of how fast&nbsp;you can run Ruby code, the choke point is often going to be the C API&nbsp;emulation, since it will require a lot of handle-juggling and indirection&nbsp;compared to MRI. So without supporting the C API, there's a very large&nbsp;part of the story missing...a part of the story that accesses frame&nbsp;locals, closure bodies, bindings, and so on.<br /><br />Of course if you can run Ruby code as fast as C, maybe it won't&nbsp;matter. :) Users can just implement their extensions in Ruby.&nbsp;JRuby is starting to approach that kind of performance for non-numeric,&nbsp;non-closure cases, but that sort of perf is not yet widespread enough to&nbsp;bank on.<br /><h4 style="text-align: left;">Ruby 1.9 encoding support</h4>Any benchmark that touches anything relating to binary text&nbsp;data must have encoding support, or you're really fudging the&nbsp;numbers. Encoding touches damn near everything, and can add a significant amount of overhead to String-manipulating benchmarks.<br /><h4 style="text-align: left;">Garbage collection and object allocation</h4>It's easy for a new impl to show good performance on benchmarks that&nbsp;do no allocation (or little allocation) and require no GC, like raw numerics (fib, tak, etc).&nbsp;Macruby and Rubinius, for example, really shine here. But many impls&nbsp;have drastically different performance when an algorithm starts&nbsp;allocating objects.&nbsp;Very&nbsp;few applications are doing pure integer numeric algorithms, so object<br />allocation and GC performance are an absolutely critical part of the performance story.<br /><h4 style="text-align: left;">Concurrency / Parallelism</h4>If you intend to be an impl that supports parallel thread execution,&nbsp;you're going to have to deal with various issues before publishing&nbsp;numbers. For example, threads can #kill or #raise each other, which in<br />a truly parallel runtime requires periodic safepoints/pings to know&nbsp;whether a cross-thread event has fired. If you're not handling those&nbsp;safepoints, you're not telling the whole story, since they impact execution.<br /><br />There's also the thread-safety of runtime structures to be considered. As an example,&nbsp;Rubinius until recently had a hard lock around a data structure responsible for invalidating call sites, which&nbsp;meant that its simple inline cache could see a severe performance&nbsp;degradation at polymorphic call sites (they've since added polymorphic caching to ameliorate this case). The thread-safety of a Ruby implementation's core runtime structures can drastically impact even straight-line, non-concurrent performance.<br /><br />Of course, for an impl that doesn't support parallel execution (which&nbsp;would put it in the somewhat more limited realm of MRI), you can get away with GIL&nbsp;scheduling tricks. You just won't have a very good in-process scaling story.<br /><h4 style="text-align: left;">Tracing/debugging</h4>All current impls support tracing or debugging APIs, though some (like<br />JRuby) require you to enable support for them via command-line or compile-time flags. A Ruby implementation needs to have an answer for&nbsp;this, since the runtime-level hooks required will have an impact...and may&nbsp;require users to opt-in.<br /><h4>ObjectSpace</h4>ObjectSpace#each_object needs to be addressed before talking about&nbsp;performance. In JRuby, supporting each_object over arbitrary types was&nbsp;a major performance issue, since we had to track all objects in a&nbsp;separate data structure in case they were needed. We ultimately&nbsp;decided each_object would only work with Class and Module, since those&nbsp;were the major practical use cases (and tracking Class/Module hierarchies is far easier than tracking all objects in the system).<br /><br />Depending on how a Ruby implementation tracks in-memory objects (and depending on the level of accuracy expected from ObjectSpace#each_object) this can impact how allocation logic and GC are optimized.<br /><h4 style="text-align: left;">Method invalidation</h4>Several implementations can see severe global effects due to methods like Object#extend&nbsp;blowing all global caches (or at least several caches), so you need to be&nbsp;able to support #extend in a reasonable way before talking about&nbsp;performance. Singleton objects also have a similar effect, since they&nbsp;alter the character of method caches by introducing new anonymous types at&nbsp;any time (and sometimes, in rapid succession).<br /><br />In JRuby, singleton and #extend effects are limited to the call sites that see them. I also have an experimental branch that's smarter about type identity, so simple anonymous types (that have only had modules included or extended into them) will not damage caches at all. Hopefully we'll land that in a future release.<br /><h4 style="text-align: left;">Constant lookup and invalidation</h4>I believe all implementations have implemented constant cache&nbsp;invalidation as a global invalidation, though there are other more&nbsp;complicated ways to do it. The main challenge is the fact that constant lookup is tied to both lexical scope and class hiearchy, so invalidating individual constant lookup sites is usually infeasible. Constant lookup is also rather tricky&nbsp;and must be implemented correctly&nbsp;before talking about the performance of any benchmark that references&nbsp;constants.<br /><h4 style="text-align: left;">Rails</h4><div>Finally, regardless of how awesome a new Ruby implementation claims to be, most users will simply ask "but does it run Rails?" You can substitute your favorite framework or library, if you like...the bottom line is that an awesome Ruby implementation that doesn't run any Ruby applications is basically useless. Beware of crowing about your victory over Ruby performance before you can run code people actually care about.</div></div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/6804148115098747648/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2012/10/so-you-want-to-optimize-ruby.html#comment-form' title='17 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/6804148115098747648'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/6804148115098747648'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2012/10/so-you-want-to-optimize-ruby.html' title='So You Want To Optimize Ruby'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>17</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-5284790001981835990</id><published>2012-09-26T14:03:00.003-07:00</published><updated>2012-09-26T14:06:40.428-07:00</updated><title type='text'>Explanation of Warnings From MRI's Test Suite</title><content type='html'><div dir="ltr" style="text-align: left;" trbidi="on">JRuby has, for some time now, run the same <a href="https://github.com/jruby/jruby/tree/master/test/externals/ruby1.9">test suite as MRI</a> (C Ruby, Matz's Ruby). Because not all tests pass, we use <a href="https://github.com/seattlerb/minitest-excludes">minitest-excludes</a> to mask out the failures, and over time we unmask stuff as we fix it.<br /><br />However, there's a number of warnings we get from the suite that are nonfatal and unmaskable. I thought I'd show them to you and tell their stories.<br /><br /><b>JRuby 1.9 mode only supports the `psych` YAML engine; ignoring `syck`</b><br /><b><br /></b><div>When we started implementing support for the new "psych" YAML engine that Aaron Patterson created (atop libyaml) for Ruby 1.9, we decided that we would not support the broken "syck" engine anymore. The libyaml version is strictly YAML spec compliant, and this is our contribution to ridding the world of "syck"'s broken YAML forever.<br /><br /><b>GC.stress= does nothing on JRuby</b><br /><b><br /></b></div><div>JRuby does not have direct control over the JVM's GC, and so we can't implement things like GC.stress=, which MRI uses to put the GC into "stress" mode (GCing much more frequently to better test GC stability and behavior). There are flags for the JVM to do this sort of testing, but since we don't really need to test the JVM's GC for correctness and stability, we have not exposed those flags directly.</div><div><br /></div><div>This flag is used in a number of MRI tests to force GC to happen more often and/or to actually test GC behaviors.<br /><br /><b>SAFE levels are not supported in JRuby</b><br /><b><br /></b></div><div>JRuby does not support standard Ruby's security model, "safe levels", because we believe safe levels are a flawed, too-coarse mechanism. On JRuby, you can use standard Java security policies.</div><div><br /></div><div>We have debated mapping the various Ruby safe levels to equivalent sets of Java security permissions, but have never gotten around to it.<br /><br /><b>GC.enable does nothing on JRuby / GC.disable does nothing on JRuby</b><br /><b><br /></b></div><div>There's no standard API on the JVM to disable the garbage collector completely, so GC.enable and GC.disable do nothing in JRuby.</div><div><br /></div><div>It's also interesting to note that while you <b>can</b>&nbsp;request a GC run from the JVM by calling System.gc, JRuby also stubs out Ruby's GC.start. We opted to do this because GC.start is used in some Ruby libraries as a band-aid around Ruby's sometimes-slow GC, but the same call on JRuby is both unnecessary (because GC overhead is rarely a problem) and a major performance hit (because it triggers a full GC over the entire heap).</div><div><br /></div></div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/5284790001981835990/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2012/09/explanation-of-warnings-from-mris-test.html#comment-form' title='0 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5284790001981835990'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5284790001981835990'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2012/09/explanation-of-warnings-from-mris-test.html' title='Explanation of Warnings From MRI's Test Suite'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>0</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-4242550722639887199</id><published>2012-09-16T21:36:00.000-07:00</published><updated>2012-09-16T21:46:42.441-07:00</updated><title type='text'>An experiment in static compilation of Ruby: FASTRUBY!</title><content type='html'><div dir="ltr" style="text-align: left;" trbidi="on">While at GoGaRuCo this weekend, I finally made good on an experiment I had been thinking about for a while: <a href="https://github.com/headius/fastruby">a static compiler for Ruby</a>. I thought I'd share it with you good people today.<br /><br />First we have a simple Ruby script with a class in it:<br /><br /><script src="https://gist.github.com/3735400.js?file=hello.rb"></script><br /><br />We compile it with fastruby, and it produces two .java source files: Hello.java and RObject.java.<br /><br />Hello.java implements the methods the Ruby class does in the script, and calls the same methods (with some mangling for invalid Java method names like _plus_ and _lt_).<br /><br /><script src="https://gist.github.com/3735400.js?file=Hello.java"></script><br /><br />RObject.java implements stubs for <u>all</u> method names seen in the script. As a result, all dynamic calls can just be virtual invocations against RObject. Classes that implement one of the methods will just work and the call is direct. Classes that don't implement the called method will raise an error.<br /><br /><script src="https://gist.github.com/3735400.js?file=RObject.java"></script><br /><br />RKernel comes with fastruby, and provides Kernel-level methods like "puts", plus methods for coercing to Java types like toBoolean and toString. It also caches some built-in singleton values like nil.<br /><br /><script src="https://gist.github.com/3735400.js?file=RKernel.java"></script><br /><br /><br />And there's a few other classes for this script to work. It should be easy to see how we could fill them out to do everything the equivalent Ruby classes do.<br /><div><br /></div><div><script src="https://gist.github.com/3735400.js?file=RFixnum.java"></script></div><div><br /></div><div><script src="https://gist.github.com/3735400.js?file=RString.java"></script></div><div><br /></div><div><script src="https://gist.github.com/3735400.js?file=RBoolean.java"></script></div><div><br /></div><div>I don't have any support for a "main" method yet, so I wrote a little runner script to test it.</div><div><br /></div><div><script src="https://gist.github.com/3735400.js?file=HelloRunner.java"></script></div><div><br /></div><div>And away we go!</div><div><br /></div><div><script src="https://gist.github.com/3735400.js?file=output.txt"></script></div><div><br /></div><div><div>This is about 30% faster than JRuby with invokedynamic. It is not doing any boundschecking (for rolling over to Bignum) but it is also not caching 1...256 Fixnum objects like JRuby does, nor caching them in any calls along the way (note that it creates three new RFixnums for every recursion that JRuby would not recreate). I call that pretty good.</div><div><br /></div><div>Obviously because this is designed to compile the whole system at once, we could also emit optimized versions of methods that look like they're doing math. That is yet to come, if I continue this little experiment at all.<br /><br />There's also some fun possibilities here. By specifying Java types, the compiler could add normal Java methods. Implementing interfaces could be done directly. And Android applications built with this tool would be entirely statically optimizable, only shipping the small amount of code they actually call and having a very minimal runtime.</div><div><br /></div><div>Pretty neat?</div></div></div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/4242550722639887199/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2012/09/an-experiment-in-static-compilation-of.html#comment-form' title='24 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/4242550722639887199'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/4242550722639887199'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2012/09/an-experiment-in-static-compilation-of.html' title='An experiment in static compilation of Ruby: FASTRUBY!'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>24</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-7858745721921352272</id><published>2012-09-04T01:00:00.002-07:00</published><updated>2012-09-04T01:02:34.459-07:00</updated><title type='text'>Avoiding Hash Lookups in a Ruby Implementation</title><content type='html'><div dir="ltr" style="text-align: left;" trbidi="on">I had an interesting realization tonight: I'm terrified of hash tables. Specifically, my work on JRuby (and even more directly, my work optimizing JRuby) has made me terrified to ever consider using a hash table in the hot path of any program or piece of code if there's any possibility of eliminating it. And what I've learned over the years is that the vast majority of execution-related (as opposed to data-related, purely dynamic-sourced lookup tables) hash tables are totally unnecessary.<br /><br />Some background might be interesting here.<br /><h2 style="text-align: left;">Hashes are a Language Designer's First Tool</h2><div>Anyone who's ever designed a simple language knows that pretty much everything you do is trivial to implement as a hash table. Dynamically-expanding tables of functions or methods? Hash table! Variables? Hash table! Globals? Hash table!</div><div><br /></div><div>In fact, some languages never graduate beyond this phase and remain essentially gobs and gobs of hash tables even in fairly recent implementations. I won't name your favorite language here, but I will name one of mine: Ruby.</div><h2 style="text-align: left;">Ruby: A Study in Hashes All Over the Freaking Place</h2><div>As with many dynamic languages, early (for some definition of "early") implementations of Ruby used hash tables all over the place. Let's just take a brief tour through the many places hash tables are used in Ruby 1.8.7</div><div><br /></div><div>(Author's note: 1.8.7 is now, by most measures, the "old" Ruby implementation, having been largely supplanted by the 1.9 series which boasts a "real" VM and optimizations to avoid most hot-path hash lookup.)</div><div><br /></div><div>In Ruby (1.8.7), all of the following are (usually) implemented using hash lookups (and of these, many are hash lookups nearly every time, without any caching constructs):</div><div><ul style="text-align: left;"><li>Method Lookup: Ruby's class hierarchy is essentially a tree of hash tables that contain, among other things, methods. Searching for a method involves searching the target object's class. If that fails, you must search the parent class, and so on. In the absence of any sort of caching, this can mean you search all the way up to the root of the hierarchy (Object or Kernel, depending what you consider root) to find the method you need to invoke. This is also known as "slow".</li><li>Instance Variables: In Ruby, you do not declare ahead of time what variables a given class's object instances will contain. Instead, instance variables are allocated as they're assigned, like a hash table. And in fact, most Ruby implementations still use a hash table for variables under some circumstances, even though most of these variables can be statically determined ahead of time or dynamically determined (to static ends) at runtime.</li><li>Constants: Ruby's constants are actually "mostly" constant. They're a bit more like "const" in C, assignable once and never assignable again. Except that they <b>are</b>&nbsp;assignable again through various mechanisms. In any case, constants are also not declared ahead of time and are not purely a hierarchically-structured construct (they are both lexically and hierarchically scoped), and as a result the simplest implementation is a hash table (or chains of hash tables), once again.</li><li>Global Variables: Globals are frequently implemented as a top-level hash table even in modern, optimized language. They're also evil and you shouldn't use them, so most implementations don't even bother making them anything other than a hash table.</li><li>Local Variables: Oh yes, Ruby has not been immune to the greatest evil of all: purely hash table-based local variables. A "pure" version of Python would have to do the same, although in practice no implementations really support that (and yes, you can manipulate the execution frame to gain "hash like" behavior for Python locals, but you must surrender your Good Programmer's Card if you do). In Ruby's defense, however, hash tables were only ever used for closure scopes (blocks, etc), and no modern implementations of Ruby use hash tables for locals in any way.</li></ul><div>There are other cases (like class variables) that are less interesting than these, but this list serves to show how easy it is for a language implementer to fall into the "everything's a hash, dude!" hole, only to find they have an incredibly flexible and totally useless language. Ruby is not such a language, and almost all of these cases can be optimized into largely static, predictable code paths with nary a hash calculation or lookup to be found.</div></div><div><br /></div><div>How? I'm glad you asked.</div><h2 style="text-align: left;">JRuby: The Quest For Fewer Hashes</h2><div>If I were to sum up the past 6 years I've spent optimizing JRuby (and learning how to optimize dynamic languages) it would be with the following phrase: Get Rid Of Hash Lookups.</div><div><br /></div><div>When I tweeted about this realization yesterday, I got a few replies back about better hashing algorithms (e.g. "perfect" hashes) and a a few replies from puzzled folks ("what's wrong with hashes?"), which made me realize that it's not always apparent how unnecessary most (execution-related) hash lookups really are (and from now on, when I talk about unnecessary or optimizable hash lookups, I'm talking about execution-related hash lookups; you data folks can get off my back right now).</div><div><br /></div><div>So perhaps we should talk a little about why hashes are bad in the first place.</div><h2 style="text-align: left;">What's Wrong With a Little Hash, Bro?</h2><div>The most obvious problem with using hash tables is the mind-crunching frustration of finding THE PERFECT HASH ALGORITHM. Every year there's a new way to calculate String hashes, for example, that's [ better | faster | securer | awesomer ] than all precedents. JRuby, along with many other languages, actually released a security fix last year to patch the great hash collision DoS exploit so many folks made a big deal about (while us language implementers just sighed and said "maybe you don't actually want a hash table here, kids"). Now, the implementation we put in place has again been "exploited" and we're told we need to move to cryptographic hashing. Srsly? How about we just give you a crypto-awesome-mersenne-randomized hash impl you can use for all your outward-facing hash tables and you can leave us the hell alone?</div><div><br /></div><div>But I digress.</div><div><br /></div><div>Obviously the cost of calculating hash codes is the first sin of a hash table. The second sin is deciding how, based on that hash code, you will distribute buckets. Too many buckets and you're wasting space. Too few and you're more likely to have a collision. Ahh, the intricate dance of space and time plagues us forever.</div><div><br /></div><div>Ok, so let's say we've got some absolutely smashing hash algorithm and foresight enough to balance our buckets so well we make Lady Justice shed a tear. We're still screwed, my friends, because we've almost certainly defeated the prediction and optimization capabilities of our VM or our M, and we've permanently signed over performance in exchange for ease of implementation.</div><div><br /></div><div>It is conceivable that a really good machine can learn our hash algorithm really well, but in the case of string hashing we still have to walk <b>some</b>&nbsp;memory to give us reasonable assurance of unique hash codes. So there's performance sin #1 violated: never read from memory.</div><div><br /></div><div>Even if we ignore the cost of calculating a hash code, which at worst requires reading some object data from memory and at best requires reading a cached hash code from elsewhere in memory, we have to contend with how the buckets are implemented. Most hash tables implement the buckets as either of the typical list forms: an array (contiguous memory locations in a big chunk, so each element must be dereferenced...O(1) complexity) or a linked list (one entry chaining to the next through some sort of memory dereference, leading to O(N) complexity for searching collided entries).</div><div><br /></div><div>Assuming we're using simple arrays, we're <b>still</b>&nbsp;making life hard for the machine since it has to see through at least one and possibly several mostly-opaque memory references. By the time we've got the data we're after, we've done a bunch of memory-driven calculations to find a chain of memory dereferences. And you wanted this to be fast?</div><h2 style="text-align: left;">Get Rid Of The Hash</h2><div>Early attempts (of mine and others) to optimize JRuby centered around making hashing as cheap as possible. We made sure our tables only accepted interned strings, so we could guarantee they'd already calculated and cached their hash values. We used the "programmer's hash", switch statements, to localize hash lookups closer to the code performing them, rather than trying to balance buckets. We explored complicated implementations of hierarchical hash tables that "saw through" to parents, so we could represent hierarchical method table relationships in (close to) O(1) complexity.</div><div><br /></div><div>But we were missing the point. The problem was in our representing any of these language features as hash tables to begin with. And so we started working toward the implementation that has made JRuby actually become the fastest Ruby implementation: eliminate all hash lookups from hot execution paths.</div><div><br /></div><div>How? Oh right, that's what we were talking about. I'll tell you.</div><h2 style="text-align: left;">Method Tables</h2><div>I mentioned earlier that in Ruby, each class contains a method table (a hash table from method name to a piece of code that it binds) and method lookup proceeds up the class hierarchy. What I didn't tell you is that both the method tables and the hierarchy are mutable at runtime.</div><div><br /></div><div>Hear that sound? It's the static-language fanatics' heads exploding. Or maybe the "everything must be mutable always forever or you are a very bad monkey" fanatics. Whatever.</div><div><br /></div><div>Ruby is what it is, and the ability to mix in new method tables and patch existing method tables at runtime is part of what makes it attractive. Indeed, it's a huge part of what made frameworks like Rails possible, and also a huge reason why other more static (or more reasonable, depending on how you look at it) languages have had such difficulty replicating Rails' success.</div><div><br /></div><div>Mine is not to reason why. Mine is but to do and die. I have to make it fast.</div><div><br /></div><div>Proceeding from the naive implementation, there are certain truths we can hold at various times during execution:</div><div><ul style="text-align: left;"><li>Most method table and hierarchy manipulation will happen early in execution. This was true when I started working on JRuby and it's largely true now, in no small part due to the fact that optmizing method tables and hierarchies that are wildly different all the time is really, really hard (so no implementer does it, so no user should do it). Before you say it: even prototype-based languages like Javascript that appear to have no fixed structure do indeed settle into a finite set of predictable, optimizable "shapes" which VMs like V8 can take advantage of.</li><li>When changes do happen, they only affect a limited set of observers. Specifically, only call sites (the places where you actually make calls in code) need to know about the changes, and even they only need to know about them if they've already made some decision based on the old structure.</li></ul><div>So we can assume method hierarchy structure is mostly static, and when it isn't there's only a limited set of cases where we care. How can we exploit that?</div></div><div><br /></div><div>First, we implement what's called an "inline cache" at the call sites. In other words, every place where Ruby code makes a method call, we keep a slot in memory for the most recent method we looked up. In another quirk of fate, it turns out most calls are "monomorphic" ("one shape") so caching more than one is <b>usually</b>&nbsp;not beneficial.</div><div><br /></div><div>When we revisit the cache, we need to know we've still got the right method. Obviously it would be stupid to do a full search of the target object's class hierarchy all over again, so what we want is to simply be able to examine the type of the object and know we're ok to use the same method. In JRuby, this is (usually) done by assigning a unique serial number to every class in the system, and caching that serial number along with the method at the call site.</div><div><br /></div><div>Oh, but wait...how do we know if the class or its ancestors have been modified?</div><div><br /></div><div>A simple implementation would be to keep a single global serial number that gets spun every time any method table or class hierarchy anywhere in the system is modified. If we assume that those changes eventually stop, this is good enough; the system stabilizes, the global serial number never changes, and all our cached methods are safely tucked away for the machine to branch-predict and optimize to death. This is how Ruby 1.9.3 optimizes inline caches (and I believe Ruby 2.0 works the same way).</div><div><br /></div><div>Unfortunately, our perfect world isn't quite so perfect. Methods do get defined at runtime, especially in Ruby where people often create one-off "singleton methods" that only redefine a couple methods for very localized use. We don't want such changes to blow all inline caches everywhere, do we?</div><div><br /></div><div>Let's split up the serial number by method name. That way, if you are only redefining the "foobar" method on your singletons, only inline caches for "foobar" calls will be impacted. Much better! This is how Rubinius implements cache invalidation.</div><div><br /></div><div>Unfortunately again, it turns out that the methods people override on singletons are very often common methods like "hash" or "to_s" or "inspect", which means that a purely name-based invalidator still causes a large number of call sites to fail. Bummer.</div><div><br /></div><div>In JRuby, we went through the above mechanisms and several others, finally settling on one that allows us to only ever invalidate the call sites that <b>actually</b>&nbsp;called a given method against a given type. And it's actually pretty simple: we spin the serial numbers on the individual classes, rather than in any global location.</div><div><br /></div><div>Every Ruby class has one parent and zero or more children. The parent connection is obviously a hard link, since at various points during execution we need to be able to walk up the class hierarchy. In JRuby, we also add a <b>weak</b>&nbsp;link from parents to children, updated whenever the hierarchy changes. This allows changes anywhere in a class hiearchy to cascade down to all children, localizing changes to just that subhierarchy rather than inflicting its damage upon more global scopes.</div><div><br /></div><div>Essentially, by actively invalidating down-hierarchy classes' serial numbers, we automatically know that matching serial numbers at call sites mean the cached method is 100% ok to use. We have reduced O(N) hierarchically-oriented hash table lookups to a single identity check. Victory!</div><h2 style="text-align: left;">Instance Variables</h2><div>Optimizing method lookups actually turned out to be the easiest trick we had to pull. Instance variables defied optimization for a good while. Oddly enough, most Ruby implementations stumbled on a reasonably simple mechanism at the same time.</div><div><br /></div><div>Ruby instance variables can be thought of as C++ or Java fields that only come into existence at runtime, when code actually starts using them. And where C++ and Java fields can be optimized right into the object's structure, Ruby instance variables have typically been implemented as a hash table that can grow and adapt to a running program as it runs.</div><div><br /></div><div>Using a hash table for instance variables has some obvious issues:</div><div><ul style="text-align: left;"><li>The aforementioned performance costs of using hashes</li><li>Space concerns; a collection of buckets already consumes space for some sort of table, and too many buckets means you are using <b>way</b>&nbsp;more space per object than you want</li></ul><div>At first you might think this problem can be tackled exactly the same way as method lookup, but you'd be wrong. What do we cache at the call site? It's not code we need to keep close to the point of use, it's the steps necessary to reach a point in a given object where a value is stored (ok, that could be considered code...just bear with me for a minute).</div></div><div><br /></div><div>There are, however, truths we can exploit in this case as well.</div><div><ul style="text-align: left;"><li>A given class of objects will generally reference a small, finite number of variable names during the lifetime of a given program.</li><li>If a variable is accessed once, it is very likely to be accessed again.</li><li>The set of variables used by a particular class of objects is largely unique to that class of objects.</li><li>The majority of the variables ever to be accessed can be determined by inspecting the code contained in that class and its superclasses.</li></ul><div>This gives us a lot to work with. Since we can localize the set of variables to a given class, that means we can store something at the class level. How about the actual layout of the values in object instances of that class?</div></div><div><br /></div><div>This is how most current implementations of Ruby actually work.</div><div><br /></div><div>In JRuby, as instance variables are first assigned, we bump a counter on the class that indicates an offset into an instance variable table associated with instances of that class. Eventually, all variables have been encountered and that table and that counter stop changing. Future instances of those objects, then, know exactly how larger the table needs to be and which variables are located where.</div><div><br /></div><div>Invalidation of a given instance variable "call site" is then once again a simple class identity check. If we have the same class in hand, we know the offset into the object is guaranteed to be the same, and therefore we can go straight in without doing any hash lookup whatsoever.</div><div><br /></div><div>Rubinius does things a little differently here. Instead of tracking the offsets at runtime, the Rubinius VM will examine all code associated with a class and use that to make a guess about how many variables will be needed. It sets up a table on the class ahead of time for those statically-determined names, and allocates exactly as much space for the object's header + those variables in memory (as opposed to JRuby, where the object and its table are two separate objects). This allows Rubinius to pack those known variables into a tighter space without hopping through the extra dereference JRuby has, and in many cases, this can translate to faster access.</div><div><br /></div><div>However, both cases have their failures. In JRuby's version, we pay the cost of a second object (an array of values) and a pointer dereference to reach it, even if we can cache the offset 100% successfully at the call site. This translates to larger memory footprints and somewhat slower access times. In Rubinius, variables that are dynamically allocated fall back on a simple hash table, so dynamically-generated (or dynamically-mutated) classes may end up accessing some values in a much slower way than others.</div><div><br /></div><div>The quest for perfect Ruby instance variable tables continues, but at least we have the tools to almost completely eliminate hashes right now.</div><h2 style="text-align: left;">Constants</h2><div>The last case I'm going to cover in depth is that of "constant" values in Ruby.</div><div><br /></div><div>Constants are, as I mentioned earlier, stored on classes in another hash table. If that were their only means of access, they would be uninteresting; we could use exactly the same mechanism for caching them as we do for methods, since they'd follow the same structure and behavior (other than being somewhat more static than method tables). Unfortunately, that's not the case; constants are located based on both lexical and hierarchical searches.</div><div><br /></div><div>In Ruby, if you define a class or module, all constants lexically contained in that type's enclosing scopes are also visible within the type. This makes it possible to define new lexically-scoped aliased for values that might otherwise be difficult to retrieve without walking a class hierarchy or requiring a parent/child relationship to make those aliases visible. It also defeats nearly all reasonable mechanisms for eliminating hash lookups.</div><div><br /></div><div>When you access a constant in Ruby, the implementation must first search all lexically-enclosing scopes. Each scope has a type (class or module) associated, and we check that type (and not its parents) for the constant name in question. Failing that, we fall back on the current type's class hierarchy, searching all the way up to the root type. Obviously, this could be far more searching than even method lookup, and we want to eliminate it.</div><div><br /></div><div>If we had all the space in the world and no need to worry about dangling references, using our down-hierarchy method table invalidation would actually work very well here. We'd simply add another hierarchy for invalidation: lexical scopes. In practice, however, this is not feasible (or at least I have not found a way to make it feasible) since there are <b>many times</b>&nbsp;more lexical scopes in a given system than there are types, and a large number of those scopes are transient; we'd be tracking thousands or tens of thousands of parent/child relationships weakly all over the codebase. Even worse, invalidation due to constant updates or hierarchy changes would have to proceed both down the class hierarchy and throughout all lexically-enclosing scopes in the entire system. Ouch!</div><div><br /></div><div>The current state of the art for Ruby implementations is basically our good old global serial number. Change a constant anywhere in Ruby 1.9.3, Rubinius, or JRuby, and you have just caused all constant access sites to invalidate (or they'll invalidate next time they're encountered). Now this sounds bad, perhaps because I told you it was bad above for method caching. But remember that the majority of Ruby programmers advise and practice the art of keeping constants...constant. Most of the big-name Ruby folks would call it a bug if your code is continually assigning or reassigning constants at runtime; there are other structures you could be using that are better suited to mutation, they might say. And in general, most modern Ruby libraries and frameworks do keep constants constant.</div><div><br /></div><div>I'll admit we could do better here, especially if the world changed such that mutating constants was considered proper and advisable. But until that happens, we have again managed to eliminate hash lookups by caching values based on a (hopefully rarely modified) global serial number.</div><h2 style="text-align: left;">The Others</h2><div>I did not go into the others because the solutions are either simple or not particularly interesting.</div><div><br /></div><div>Local variables in any sane language (flame on!) are statically determinable at parse/compile time (rather than being dynamically scoped or determined at runtime). In JRuby, Ruby 1.9.3, and Rubinius, local variables are in all cases a simple tuple of offset into an execution frame and some depth at which to find the appropriate frame in the case of closures.</div><div><br /></div><div>Global variables are largely discouraged, and usually only accessed at boot time to prepare more locally-defined values (e.g. configuration or environment variable access). In JRuby, we have experimented with mechanisms to cache global variable accessor logic in a way similar to instance variable accessors, but it turned out to be so rarely useful that we never shipped it.</div><div><br /></div><div>Ruby also has another type of variable called a "class variable", which follows lookup rules almost identical to methods. We don't currently optimize these in JRuby, but it's on my to-do list.</div><h2 style="text-align: left;">Final Words</h2><div>There are of course many other ways to avoid hash lookups, with probably the most robust and ambitious being code generation. Ruby developers, JIT compiler writers, and library authors have all used code generation to take what is a mostly-static lookup table and turn it into actually-static code. But you must be careful here to not fall into the trap of simply stuffing your hash logic into a switch table; you're still doing a calculation and some kind of indirection (memory dereference or code jump) to get to your target. Analyze the situation and figure out what immutable truths there are you can exploit, and you too can avoid the evils of hashes.</div></div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/7858745721921352272/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2012/09/avoiding-hash-lookups-in-ruby.html#comment-form' title='8 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/7858745721921352272'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/7858745721921352272'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2012/09/avoiding-hash-lookups-in-ruby.html' title='Avoiding Hash Lookups in a Ruby Implementation'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>8</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-1798238886261285117</id><published>2011-10-14T13:25:00.000-07:00</published><updated>2011-10-14T13:40:29.975-07:00</updated><title type='text'>Why Clojure Doesn't Need Invokedynamic (Unless You Want It to be More Awesome)</title><content type='html'>This was originally posted as a comment on <a href="http://twitter.com/fogus">@fogus</a>'s blog post "<a href="http://blog.fogus.me/2011/10/14/why-clojure-doesnt-need-invokedynamic-but-it-might-be-nice/">Why Clojure doesn’t need invokedynamic, but it might be nice</a>". I figured it's worth a top-level post here.<br /><br />Ok, there's some good points here and a few misguided/misinformed positions. I'll try to cover everything.<br /><br />First, I need to point out a key detail of invokedynamic that may have escaped notice: any case where you must bounce through a generic piece of code to do dispatch -- regardless of how fast that bounce may be -- prevents a whole slew of optimizations from happening. This might affect Java dispatch, if there's any argument-twiddling logic shared between call sites. It would definitely affect multimethods, which are using a hand-implemented PIC. Any case where there's intervening code between the call site and the target would benefit from invokedynamic, since invokedynamic could be used to plumb that logic and let it inline straight through. This is, indeed, the primary benefit of using invokedynamic: arbitrarily complex dispatch logic folds away allowing the dispatch to optimize as if it were direct.<br /><br />Your point about inference in Java dispatch is a fair one...if Clojure is able to infer all cases, then there's no need to use invokedynamic at all. But unless Clojure is able to infer all cases, then you've got this little performance time bomb just waiting to happen. Tweak some code path and obscure the inference, and kablam, you're back on a slow reflective impl. Invokedynamic would provide a measure of consistency; the only unforeseen perf impact would be when the dispatch turns out to *actually* be polymorphic, in which case even a direct call wouldn't do much better.<br /><br />For multimethods, the benefit should be clear: the MM selection logic would be mostly implemented using method handles and "leaf" logic, allowing hotspot to inline it everywhere it is used. That means for small-morphic MM call sites, all targets could potentially inline too. That's impossible without invokedynamic unless you generate every MM path immediately around the eventual call.<br /><br />Now, on to defs and Var lookup. Depending on the cost of Var lookup, using a SwitchPoint-based invalidation plus invokedynamic could be a big win. In Java 7u2, SwitchPoint-based invalidation is essentially free until invalidated, and as you point out that's a rare case. There would essentially be *no* cost in indirecting through a var until that var changes...and then it would settle back into no cost until it changes again. Frequently-changing vars could gracefully degrade to a PIC.<br /><br />It's also dangerous to understate the impact code size has on JVM optimization. The usual recommendation on the JVM is to move code into many small methods, possibly using call-through logic as in multimethods to reuse the same logic in many places. As I've mentioned, that defeats many optimizations, so the next approach is often to hand-inline logic everywhere it's used, to let the JVM have a more optimizable view of the system. But now we're stepping on our own feet...by adding more bytecode, we're almost certainly impacting the JVM's optimization and inlining budgets.<br /><br />OpenJDK (and probably the other VMs too) has various limits on how far it will go to optimize code. A large number of these limits are based on the bytecoded size of the target methods. Methods that get too big won't inline, and sometimes won't compile. Methods that inline a lot of code might not get inlined into other methods. Methods that inline one path and eat up too much budget might push out more important calls later on. The only way around this is to reduce bytecode size, which is where invokedynamic comes in.<br /><br />As of OpenJDK 7u2, MethodHandle logic is not included when calculating inlining budgets. In other words, if you push all the Java dispatch logic or multimethod dispatch logic or var lookup into mostly MethodHandles, you're getting that logic *for free*. That has had a tremendous impact on JRuby performance; I had previous versions of our compiler that did indeed infer static target methods from the interpreter, but they were often *slower* than call site caching solely because the code was considerably larger. With invokedynamic, a call is a call is a call, and the intervening plumbing is not counted against you.<br /><br />Now, what about negative impacts to Clojure itself...<br /><br />#0 is a red herring. JRuby supports Java 5, 6, and 7 with only a few hundred lines of changes in the compiler. Basically, the compiler has abstract interfaces for doing things like constant lookup, literal loading, and dispatch that we simply reimplement to use invokedynamic (extending the old non-indy logic for non-indified paths). In order to compile our uses of invokedynamic, we use Rémi Forax's JSR-292 backport, which includes a "mock" jar with all the invokedynamic APIs stubbed out. In our release, we just leave that library out, reflectively load the invokedynamic-based compiler impls, and we're off to the races.<br /><br />#1 would be fair if the Oracle Java 7u2 early-access drops did not already include the optimizations that gave JRuby those awesome numbers. The biggest of those optimizations was making SwitchPoint free, but also important are the inlining discounting and MutableCallSite improvements. The perf you see for JRuby there can apply to any indirected behavior in Clojure, with the same perf benefits as of 7u2.<br /><br />For #2, to address the apparent vagueness in <a href="http://blog.headius.com/2011/08/invokedynamic-in-jruby-constant-lookup.html">my blog post</a>...the big perf gain was largely from using SwitchPoint to invalidate constants rather than pinging a global serial number. Again, indirection folds away if you can shove it into MethodHandles. And it's pretty easy to do it.<br /><br />#3 is just plain FUD. Oracle has committed to making invokedynamic work well for Java too. The current thinking is that "lambda", the support for closures in Java 7, will use invokedynamic under the covers to implement "function-like" constructs. Oracle has also committed to Nashorn, a fully invokedynamic-based JavaScript implementation, which has many of the same challenges as languages like Ruby or Python. I talked with Adam Messinger at Oracle, who explained to me that Oracle chose JavaScript in part because it's so far away from Java...as I put it (and he agreed) it's going to "keep Oracle honest" about optimizing for non-Java languages. Invokedynamic is driving the future of the JVM, and Oracle knows it all too well.<br /><br />As for #4...well, all good things take a little effort :) I think the effort required is far lower than you suspect, though.</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/1798238886261285117/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2011/10/why-clojure-doesnt-need-invokedynamic.html#comment-form' title='4 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/1798238886261285117'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/1798238886261285117'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2011/10/why-clojure-doesnt-need-invokedynamic.html' title='Why Clojure Doesn't Need Invokedynamic (Unless You Want It to be More Awesome)'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>4</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-3395301669254719136</id><published>2011-08-10T11:00:00.000-07:00</published><updated>2011-08-10T11:59:32.843-07:00</updated><title type='text'>Invokedynamic in JRuby: Constant Lookup</title><content type='html'><i>This is the first of a set (not a series...there's no particular order) of articles I'll write on how JRuby is using invokedynamic. Hopefully they will show Rubyists how drastically invokedynamic is going to improve JRuby, and show other JVM language folks how to use invokedynamic effectively.</i><br /><br />Hello friends!<br /><br />I figured it's about time for me to start writing a bit on how JRuby is actually using invokedynamic.<br /><br />As of today, JRuby utilizes invokedynamic far more than any other mainstream JVM language. We have worked very closely with the JSR leads and the OpenJDK developers to make sure invokedynamic runs well. And we have been advocating invokedynamic as a game-changer for the JVM and for JVM languages.<br /><br />Let's explore one area where JRuby is using invokedynamic: Ruby's "constant" lookup.<br /><br /><b>Non-constant "Constants"</b><br /><br />A constant in Ruby is defined on a class or module, and is subject to Ruby's typical namespacing logic. Constants start with a capital letter.<br /><br />I often put "constants" in parentheses because constant values can be reassigned. This will usually produce a warning...but not an error. This means we can't simply look up constant values once and never look them up again (without special tricks I'll get into later).<br /><br />Constant lookup is a also bit more complicated than method lookup. When retrieving a constant, Ruby first scans lexically-enclosing scopes' classes and modules for the constant. If the constant can't be found, the next search walks the current class's inheritance hierarchy. If we still can't find the constant, const_missing is called on the current class.<br /><br />In order to make constant lookup fast, we want to do some sort of caching. In classic JRuby, Ruby 1.9 (YARV), Rubinius, and probably most other modern Ruby implementations, this is done with a global serial number. Whenever a constant is updated or a module is included (changing the inheritance hierarchy) all cached constants everywhere are forced to lookup again.<br /><br />I have played with mechanisms for reducing the global impact of constant invalidation, but because constants can be looked up lexically it's simply too complicated to localize (since we need invalidate classes down-hierarchy from the change <b>and</b> we also need to update all lexical scopes that might see the change).<br /><br /><b>Constant Invalidation in JRuby 1.6</b><br /><br />The logic in JRuby 1.6 goes something like this:<br /><br /><ul><li>If cache is empty or invalid, retrieve the constant value in the usual way (lexical, hierarchical search). Store the value with the current global constant serial number.</li>
|
|
94
|
+
<li>On subsequent lookups, check cache for validity against the global constant serial number. If we have a value cached and the cache is still valid, return it.</li>
|
|
95
|
+
<li>If any constant in the system is updated, or if a module is included into an existing class hierarchy, flip the serial number and force future constant lookups to re-cache.</li>
|
|
96
|
+
</ul><div>This turns out to work fairly well. The same mechanism in Ruby 1.9 produced drastically faster constant lookups, and JRuby's performance is even better than 1.9.</div><div><br /></div><div>But there's a problem here. Because there's this constant pinging of the global constant serial number, every constant access can potentially produce a new value. So we're paying the cost to check that serial number as well as interfering with optimizations that want to see constant values actually be constant.</div><div><br /></div><div>Can we do better?</div><div><br /></div><div><b>Quick Invokedynamic Primer</b></div><div><br /></div><div>The main atom of invokedynamic is the MethodHandle. Method handles are essentially function pointers, which can point at Java methods or fields, constructors, constant values, or other method handles. Invokedynamic also provides the MethodHandles utility class, which lets us juggle method handles in various ways:</div><div><ul><li>adapting method signatures by casting, adding, moving, or dropping arguments</li>
|
|
97
|
+
<li>combining three handles ("test", "target", and "fallback") to form new a "guard with test" if-statement-like handle</li>
|
|
98
|
+
<li>wrap handles with exception handling or argument/return pre/post-processing</li>
|
|
99
|
+
</ul><div>You can think of method handles and the chains of adapter handles that stitch them together as a special sort of functional language the JVM knows how to optimize. Given a chain of handles, you should usually get a piece of code that optimizes as well as (or better, in some cases) writing the same logic by hand in Java.</div></div><div><br /></div><div>The invokedynamic bytecode simply provides a place to plug a method handle chain into code. When the JVM encounters an invokedynamic bytecode, it calls a "bootstrap method" associated with that bytecode for further instructions.</div><div><br /></div><div>The bootstrap method returns a CallSite object, provided in java.lang.invoke. There are constant call sites for constant values, mutable call sites for when the target handle chain may have to change, and volatile call sites for when those changes must immediately be reflected across threads.</div><div><br /></div><div>Once a CallSite has been installed for a given invokedynamic, subsequent hits skip the bootstrapping process, and we're off to the races.</div><div><br /></div><div><b>SwitchPoint</b></div><div><b><br /></b></div><div>I mentioned that the MethodHandles class provides a "guardWithTest" method for combining a test, a target (the "then" branch), and a fallback (the "else" branch). SwitchPoint, also in java.lang.invoke, acts like an on/off guardWithTest that once turned off can never be turned on again. You provide a target and fallback, and until the "switch" is thrown the target will be invoked. After the switch is thrown the fallback will be called.</div><div><br /></div><div>What's the difference between this and a guardWithTest where the test just pings some global value? The difference is that SwitchPoint doesn't need to check anything.</div><div><br /></div><div><b>Optimization and Deoptimization in the JVM</b></div><div><b><br /></b></div><div>When the JVM decides to optimize a piece of code, it does so in an <b>optimistic</b>&nbsp;way. In very broad terms, this means it assumes its information up to this point is perfect: no new methods or classes will be introduced, profiling information is accurate, etc. Based on this "perfect" view of the world, it aggressively optimizes code.</div><div><br /></div><div>Of course, the world isn't perfect. The JVM has to give up profiling and monitoring at some point, so it always has an imperfect view of the system. In order to avoid its aggressive optimizations triggering a fatal error later on, JVMs like OpenJDK (Hotspot) do something called <b>deoptimization</b>.</div><div><br /></div><div>Deoptimization is the process by which running, optimized code can adapt on-the-fly to a changing system. In OpenJDK, there's several ways this is accomplished:</div><div><ul><li>Branches out of compiled code back into the interpreter, when compiled code is determined to be invalid.</li>
|
|
100
|
+
<li>Guards around inlined virtual method accesses, to ensure we're still calling against the same class.</li>
|
|
101
|
+
<li>On-stack replacement, for fixing up a running method already on the native call stack</li>
|
|
102
|
+
<li>...</li>
|
|
103
|
+
</ul><div>Because of this ability to deoptimize, it's possible to support zero-cost guards at the JVM level. Returning to SwitchPoint, we can see how this new form of "guardWithTest" can be basically free: we're explicitly telling the JVM this switch is a rare occurrence it can optimize aggressively.</div><div><br /></div><div><b>SwitchPoint for Constant Lookup</b></div><div><br /></div><div>JRuby on invokedynamic uses SwitchPoint for constant lookup, as you'd expect. Instead of actively pinging that global constant serial number, we instead use a global SwitchPoint object to guard all cached constant accesses. When it comes time to invalidate the system's constants, we just flip the SwitchPoint off and create a new one. All SwitchPoint-guarded constant accesses in the system must then recache and use the new SwitchPoint.</div><div><br /></div><div>In a well-behaved system, we should reach a steady state where no new constants are being defined and no new modules are being introduced. Because we're using SwitchPoint, the stable state means all constant accesses are treated as truly constant by the JVM, allowing optimizations that were impossible before. And of course this also means that we've achieved constant lookup performance very near a theoretical maximum.</div></div><div><br /></div><div><b>Numbers</b></div><div><b><br /></b></div><div>First, a caveat: SwitchPoint is implemented in a fairly naïve way in the released OpenJDK 7, using a volatile field as the switch value. As a result, SwitchPoint guardWithTest is very slow currently, and JRuby's SwitchPoint-based constant logic must be enabled. I show numbers below based on leading-edge Hotspot compiler patches that will go into the first update release (numbers provided by one of the Hotspot devs, Christian Thalinger...thanks Christian!)<br /><br />The benchmark we're running is a modified version of <a href="https://github.com/jruby/jruby/blob/master/bench/language/bench_const_lookup.rb">bench_const_lookup</a> in JRuby's benchmark suite. The modification here runs more iterations (10M instead of 1M) with more constant lookups (50 instead of 10) to get a better idea of optimized performance.</div><div><br /></div><div>Here's JRuby running our constant-lookup benchmark without SwitchPoint-based constants on Java 7:</div><div><script src="https://gist.github.com/1137667.js"> </script><br /><br />As I said before, this is pretty good. JRuby's existing constant lookup performance is roughly 2x faster than Ruby 1.9.2.<br /><br />Next, we'll try JRuby with SwitchPoint constants on Java 7 (released version, so we expect this to be slow):</div><br /><div><script src="https://gist.github.com/1137655.js"> </script><br /><br />The perf hit of purely volatile SwitchPoint is apparent.<br /><br />And finally, JRuby with SwitchPoint constants on a dev build of Hotspot, which uses deoptimization rather than a volatile field:</div><br /><div><script src="https://gist.github.com/1137650.js"> </script><br /><br />This is basically the performance of the 10M iteration loop alone. In fact, if you look at the resulting optimized assembly, the constant accesses have been <b>eliminated entirely</b>&nbsp;since they're optimistically inlined and never used. Of course this would normally not happen in real code, but it shows how much better the JVM can optimized Ruby's behavior using invokedynamic.</div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/3395301669254719136/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2011/08/invokedynamic-in-jruby-constant-lookup.html#comment-form' title='13 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/3395301669254719136'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/3395301669254719136'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2011/08/invokedynamic-in-jruby-constant-lookup.html' title='Invokedynamic in JRuby: Constant Lookup'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>13</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-5727286770456885866</id><published>2011-08-02T10:27:00.000-07:00</published><updated>2011-08-02T21:03:30.529-07:00</updated><title type='text'>JRuby and Java 7: What to Expect</title><content type='html'>Java 7 has landed, with a modest set of new features and a few major improvements as well. What can you expect from JRuby running on Java 7?<br /><br /><b>What's In Java 7</b><br /><b><br /></b><br />The biggest changes in Java 7 are not related to the Java language at all. Sure, there's the "project coin" enhancements to the Java language, which add some exception-handling shortcuts, new literals for numbers, arrays, hashes, the oft-requested "strings in switch" support, and a few other things. But they're modest incremental changes; the real revolution is at the JVM and JDK level.<br /><br /><b>Invokedynamic</b><br /><b><br /></b><br />The most important change in Java 7 is the incorporation of a new bytecode -- invokedynamic -- and an API for building chains of "method handles" to back that bytecode up.<br /><br />You can look at invokedynamic as a way for JVM users to communicate directly with the optimizing backend of the JVM. Method handles act as both function pointers and as function combinators, allowing a built-in way to construct a call protocol flow from a caller to a callee. You can move arguments around, insert new arguments, process existing arguments and return values, catch exceptions, and perform fast guarded branches between two (or more) paths. The invokedynamic bytecode itself provides a bytecode-level hook to which you attach your method handle chain, with the assumption that the JVM can optimize that chain directly into the invokedynamic caller.<br /><br />The tl;dr is that invokedynamic makes it possible for the JVM to see through complicated method call logic, such as that found in dynamic languages, and optimize that logic like it would for regular "static" calls.<br /><br />JRuby's master branch already takes heavy advantage of invokedynamic, by routing most Ruby calls through invokedynamic operations. For simple paths and those that have been optimized by the Hotspot guys (Hotspot is the VM at the core of OpenJDK), invokedynamic often provides performance improvements of 150-200%, with work ongoing to make it even faster. Other paths may not be as well-optimized by the "dot zero" version of OpenJDK 7, so there's opportunity to improve them.<br /><br />Because JRuby is already well along the road to utilizing invokedynamic, you can try it out today.<br /><br /><ol><li>Build your own JRuby from master or grab a snapshot from&nbsp;<a href="http://ci.jruby.org/snapshots">our CI server</a>.</li><li>Grab a build of <a href="http://www.oracle.com/technetwork/java/javase/downloads/java-se-jdk-7-download-432154.html">OpenJDK 7 from Oracle</a> (or a build of <a href="http://code.google.com/p/openjdk-osx-build/">OpenJDK 7 for OS X</a>).</li><li>Point JAVA_HOME at the new JDK and try out JRuby!</li></ol><div>We're looking for small benchmarks that show the performance of invokedynamic (good or bad), so please contact me, the JRuby team, or the JRuby users mailing list with your reports from the field. Also, feel free to open performance bugs on the <a href="http://bugs.jruby.org/">JRuby bug tracker</a> if invokedynamic performs <b>worse</b>&nbsp;than non-invokedynamic. Pass&nbsp;-Xcompile.invokedynamic=false to JRuby to revert to the old non-invokedynamic logic.</div><div><br /></div><div><b>NIO.2</b></div><div><b><br /></b></div><div>NIO is Java's "New IO" APIs, a set of wrappers around low-level file-descriptor logic and memory buffers. NIO has been around since Java 1.4, but the recent update -- dubbed <a href="http://nio.2/">NIO.2</a> -- brings a sorely-needed update to the functionality provided:</div><div><ul><li>Filesystem operations (like symlinks, permissions, etc) are now almost all available through NIO.2's filesystem APIs. This also includes standard, cross-platform support for filesystem events, such as watching a directory for changes (using efficient OS-level operations, rather than polling).</li><li>File and directory walking now comes with considerably less overhead and more options for filtering directory lists <b>before</b>&nbsp;handing filenames off to user code. There's also support for opening a directory directly and walking its contents as you would a file.</li><li>Most IO channel types now have asynchronous versions. Asynchronous in this case means "punt my IO operation to a built-in thread pool", with subsequent code checking on the status of those operations and getting results from a "future" handle.</li></ul><div>For JRuby, the new IO APIs will mean we can support more filesystem operations across platforms without resorting to native code. It will also provide JRuby users a means of handling filesystem events and asynchronous IO operations without using a platform-specific library. We have not yet started adding NIO.2 support to JRuby's core classes, but that will come soon.</div></div><div><br /></div><div><b>General Improvements</b></div><div><b><br /></b></div><div>There's lots of smaller, less flashy changes in OpenJDK that also appear to help JRuby.</div><div><br /></div><div>Even without invokedynamic, the latest OpenJDK 7 builds usually perform better than OpenJDK 6. Some benchmarks have proven to be as much as 2x faster, just by upgrading the JVM! General perf improvements will be more modest, but in almost every case we've tested OpenJDK 7 definitely performs better.</div><div><br /></div><div>The release of OpenJDK 7 also brings improvements to the "tiered" compilation mode. Tiered compilation aims to merge the benefits of the "client" mode (fast startup) with those of the "server" mode (maximum peak performance). You can turn on tiered compilation using -XX:+TieredCompilation (in JAVA_OPTS or at the "java" command line, or prefixed with -J when passed to JRuby). We're looking for user reports about how well "tiered" mode works, too.</div><div><br /></div><div>This general improvement means that even JRuby 1.6.x users can take advantage of OpenJDK 7 today, with the promise of even bigger improvements in JRuby 1.7 (our target release for pervasive invokedynamic support).</div><div><br /></div><div><b>Consistency</b></div><div><b><br /></b></div><div>As with previous Java releases, a great deal of care has been taken to ensure existing applications work properly. That applies as well to Java 7. We have been testing against Java 7 for over a year, on and off, and recently started running tests "green" with even heavy invokedynamic use.</div><div><br /></div><div>We have made no major Java 7-specific fixes in JRuby...it should generally "just work".</div><div><br /></div><div><b>Let Us Know!</b></div><div><b><br /></b></div><div>As always, we really want to hear from you bleeding-edge users that are playing around with JRuby on Java 7. Please don't be shy...let us know how it works for you!<br /><br /><i style="font-weight: bold;">Update:</i>&nbsp;The Hotspot guys have been helping me find invokedynamic bottlenecks in a few JRuby microbenchmarks, and discovered that a flaw in invokedynamic was causing <b>too much</b>&nbsp;code to inline, forcing out more important optimizations. The details belong in another post, but they offered me a long Hotspot flag to accomplish basically what their fix does:&nbsp;<span class="Apple-style-span" style="font-family: 'Bitstream Vera Sans Mono', Courier, monospace; font-size: 12px; line-height: 16px; white-space: pre;">-XX:CompileCommand=dontinline,org.jruby.runtime.invokedynamic.InvokeDynamicSupport::invocationFallback</span>&nbsp;... With this flag, performance on e.g. "tak" easily beats stock JRuby (see the third benchmark run here: <a href="https://gist.github.com/1121880">https://gist.github.com/1121880</a>).<br /><br />I would recommend trying this flag if you are finding invokedynamic slowdowns in JRuby.<br /></div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/5727286770456885866/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2011/08/jruby-and-java-7-what-to-expect.html#comment-form' title='22 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5727286770456885866'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5727286770456885866'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2011/08/jruby-and-java-7-what-to-expect.html' title='JRuby and Java 7: What to Expect'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>22</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-1913573676049543893</id><published>2011-07-21T14:17:00.000-07:00</published><updated>2011-07-21T14:17:38.436-07:00</updated><title type='text'>Next July, Last Friday, This Tuesday</title><content type='html'>So after months of not blogging anything technical, I'm going to blog something non-technical. Hopefully tech posts will pick up soon once my new baby boy Elliott is a bit older and less needy :)<br /><br /><b>When Is "This Friday"?</b><br /><br />The most confusing time-oriented statements (for me) are when people use "this", "next", and "last" to describe a specific day or month. Some people consider "this" to always be the day/month coming soon", and others have different meanings. This short post will describe what <i style="font-weight: bold;">I</i>&nbsp;mean, in a way that hopefully convinces you to do the same.<br /><br />If we look at what "this", "next", and "last" are modifying, a simple pattern emerges. For days of the week, they're indicating what week contains the given day. "This week" means the week we're in right now, "next week" means the week that follows this one, and "last week" means the week that preceded this one. Taking that to days of the week, then, "this Friday" should always mean "Friday of this week". Similarly, when those modifiers are applied to a month, they usually mean what year contains the given month. "This August" would mean the August of this year, and so on.<br /><br />There's no perfect way to interpret these modifiers, and even my system has some mildly confusing points.<br /><br />Let's say today is Thursday. The following day is "this Friday", as you'd expect. A week from tomorrow would be "next Friday", the Friday of next week...not tomorrow, even though that's the Friday that comes "next" in time. Perhaps a bit more confusing is using "this" to describe days in the past; "this Wednesday" would mean the day before today, since that's the Wednesday of this week. A proper sentence would be "this Wednesday I went to the store." Note the past-tense there.<br /><br />Sunday and Saturday are peculiarities too, and in almost any system they are the source of the most confusion. By my system, "this Sunday" would almost always mean a day in the past, since that's the Sunday of this week (and it would be weird to say "this Sunday" on Sunday). Similarly, "next Saturday" will almost always mean two Saturdays from now.<br /><br />Confusion about days in the past or in the future can be avoided with additional modifiers "coming" and "past". "This past Saturday" always means the Saturday nearest in the past, and "this coming Sunday" always means the next Sunday in the future. My system is not ambiguous, but adding these additional modifiers can help smooth over places where it might confuse folks unfamiliar with it.<br /><br />One alternative would be to always have "this" mean the day/month next in time, "next" to always mean the one after that, and "last" to be the one nearest in the past. But that ends up ambiguous, since if tomorrow is Friday it's unclear if "next Friday" is tomorrow or the Friday of next week.<br /><br />So, what do you think? Does this system make sense? Is there a better way to disambiguate these modifiers?</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/1913573676049543893/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2011/07/next-july-last-friday-this-tuesday.html#comment-form' title='9 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/1913573676049543893'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/1913573676049543893'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2011/07/next-july-last-friday-this-tuesday.html' title='Next July, Last Friday, This Tuesday'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>9</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-1383525904261169024</id><published>2011-03-07T11:05:00.000-08:00</published><updated>2011-03-07T11:05:23.832-08:00</updated><title type='text'>Differing java.util.regex.Matcher Unmatched Group Results on Android</title><content type='html'>Android is really an amazing little platform, but occasionally you will run into API differences. Some of these are actual bugs (like a number of reflection and enum issues in early releases), and others are just weakly-specified APIs.<br /><br />Today, I worked on <a href="http://ira.codehaus.org/browse/JRUBY-5541">JRUBY-5541</a>:&nbsp;Problem with java_import on Android (Ruboto)<br /><br />The issue boiled down to how we turn Java's camelCased method names into Ruby's snake_cased form. We were using the following code:<br /><br /><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp;private static final Pattern CAMEL_CASE_SPLITTER = Pattern.compile("(([a-z0-9])([A-Z])|([A-Za-z0-9])([A-Z][a-z]))");</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp;public static String getRubyCasedName(String javaCasedName) {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;Matcher m = CAMEL_CASE_SPLITTER.matcher(javaCasedName);</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;return m.replaceAll("$2$4_$3$5").toLowerCase();</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp;}</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;"><br /></span><br /><span class="Apple-style-span" style="font-family: inherit;">The logic here is to basically attempt two matches ORed together: methods of the form getName in the first half, and methods of the form getURLHandler in the second half. Given the resulting match, we "cleverly" did a replaceAll for both matches at the same time, combining what would be "$2_$3" for the first half and "$4_$5" in the second half.</span><br /><span class="Apple-style-span" style="font-family: inherit;"><br /></span><br /><span class="Apple-style-span" style="font-family: inherit;">This works fine against Hotspot/OpenJDK and any JVMs that use its class libraries. But Android uses Harmony's class libraries, and behaves differently. On OpenJDK, unmatched groups returned an empty string "" for the unmatched groups, properly turning "getName" and into "get_name" and "getURLHandler" into get_url_handler". On Android, however, the unmatched groups return null for the $ variables in replaceAll, causing "getName" to become "getnull_nnullame" and "getURLHandler" into something awful like "getnull_unullrlnull_hnullandler". Subsequent logic in JRuby that tried to turn methods of the form "get_name" into "name" attributes then failed to execute, causing the issue in the bug report.</span><br /><span class="Apple-style-span" style="font-family: inherit;"><br /></span><br /><span class="Apple-style-span" style="font-family: inherit;">The fix is a bit cumbersome, but not too difficult to understand: manually walk the matches and appendReplacement using only the groups that matched:</span><br /><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;"></span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp;public static String getRubyCasedName(String javaCasedName) {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;Matcher m = CAMEL_CASE_SPLITTER.matcher(javaCasedName);</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;// We do this replace loop manually because Android's Matcher produces null for unmatched $ groups.</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;// See JRUBY-5541</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;if (m.find()) {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;StringBuffer buffer = new StringBuffer();</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;m.reset();</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;while (m.find()) {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;if (m.group(2) != null) {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;// first part matched</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;m.appendReplacement(buffer, "$2_$3");</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;} else {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;// second part matched {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;m.appendReplacement(buffer, "$4_$5");</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;}</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;}</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;m.appendTail(buffer);</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;return buffer.toString().toLowerCase();</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;} else {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;return javaCasedName;</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;}</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp;}</span><br /><br />I'm not sure whether Android (Harmony) or OpenJDK is "right" in this case, since the API for Matcher.group <b>does</b>&nbsp;say it will return null for unmatched groups, but nowhere is it specified if $ variables in replace calls should do the same.</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/1383525904261169024/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2011/03/differing-javautilregexmatcher.html#comment-form' title='2 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/1383525904261169024'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/1383525904261169024'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2011/03/differing-javautilregexmatcher.html' title='Differing java.util.regex.Matcher Unmatched Group Results on Android'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>2</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-6627578193167913100</id><published>2011-02-02T10:14:00.000-08:00</published><updated>2011-02-02T15:12:27.669-08:00</updated><title type='text'>Working Around the Java Double.parseDouble Bug</title><content type='html'>You may have seen recently that Java suffers from a similar <a href="http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/">floating-point parsing bug</a> to the one that recently affected PHP users. The basic gist of it is that for this special 64-bit floating point value, the Java call Double.parseDouble("2.2250738585072012e-308") will get stuck in an infinite loop. Read the link above to understand what's happening.<br /><br />Naturally, this affects all JVM languages too, since we all use Double.parseDouble for something or another. In fact, it affects almost all the JVM language parsers and compilers (including javac itself), since they need to turn strings into doubles.<br /><br />Being the upright citizens we are on the JRuby team, we figured we'd try to beat Oracle to the punch and patch around the bug, at least for Ruby-land conversions of String to Float.<br /><br />I started by looking for calls to Double.parseDouble in JRuby. It turned out there were only two: one for the lexer, and one used by String#to_f, BigDecimal#new, and so on. That was a relief; I expected to find dozens of calls.<br /><br />It also turned out all cases had already parsed out Ruby float literal oddities, like underscores, using 'd' or 'D' for the exponentiation marker, allowing ill-formatted exponents to be treated as zero, and so on.<br /><br />My first attempt was to simply normalize the cleaned-up string and pass it to new java.math.BigDecimal(), converting that result back to a primitive double. Unfortunately, BigDecimal's constructor *also* passes through the offending Double.parseDouble code, and we're back where we started.<br /><br />Ultimately, I ended up with the following code. I make no claims this is efficient, but it appears to pass all the Float tests and specs for JRuby and does not DOS like the bad code in Double.parseDouble:<br /><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp;public static double parseDouble(String value) {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;String normalString = normalizeDoubleString(value);</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;int offset = normalString.indexOf('E');</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;BigDecimal base;</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;int exponent;</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;if (offset == -1) {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;base = new BigDecimal(value);</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;exponent = 0;</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;} else {</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;base = new BigDecimal(normalString.substring(0, offset));</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;exponent = Integer.parseInt(normalString.charAt(offset + 1) == '+' ?</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;normalString.substring(offset + 2) :</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;normalString.substring(offset + 1));</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;}</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp; &nbsp; &nbsp;return base.scaleByPowerOfTen(exponent).doubleValue();</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;">&nbsp;&nbsp; &nbsp;}</span><br /><span class="Apple-style-span" style="font-family: 'Courier New', Courier, monospace;"><br /></span><br /><span class="Apple-style-span" style="font-family: inherit;">I didn't say it was particularly clever or efficient...but there you have it. A few notes:</span><br /><ul><li>Do I really need UNLIMITED precision here? I almost used it to ensure there's no peculiarities passing through BigDecimal on the way to double, but are any such peculiarities outside 128-bit precision?</li><li>It might have been more efficient to normalize the decimal position and exponent and then see if it matched the magic value. But of course this magic value was not known until recently, so why risk there being another one?</li><li>Using BigDecimal is also lazy. I am lazy.</li></ul><div>I welcome improvements. Everyone will probably need to start using code like this, since there will be a lot of unpatched JVMs out there for a long time.<br /><br />I'm happy to say JRuby will be the first JVM language to route around the Double.parseDouble bug :)<br /><br /><i style="font-weight: bold;">Update: </i>The JRuby commit with this logic is&nbsp;<a href="https://github.com/jruby/jruby/commit/4c712963885c0117b95066d927520a6a738c2a65">4c71296</a>, and the JRuby bug is at&nbsp;<a href="http://jira.codehaus.org/browse/JRUBY-5441">http://jira.codehaus.org/browse/JRUBY-5441</a>.<br /><br /><i style="font-weight: bold;">Update:</i>&nbsp;A commented on Hacker News pointed out that BigDecimal.doubleValue actually just converts to a string and calls Double.parseDouble. So unfortunately, the mechanism above only worked in an earlier version where I was losing some precision by calling Math.pow(10, exponent) rather than scaleByPowerOfTen. The version above unfortunately does <b>not work</b>, so it's back to the drawing board. C'est la vie!</div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/6627578193167913100/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2011/02/working-around-java-doubleparsedouble.html#comment-form' title='13 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/6627578193167913100'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/6627578193167913100'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2011/02/working-around-java-doubleparsedouble.html' title='Working Around the Java Double.parseDouble Bug'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>13</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-8236105658793522179</id><published>2011-01-19T12:31:00.000-08:00</published><updated>2011-01-25T21:44:30.183-08:00</updated><title type='text'>JRuby on Rails on Amazon Elastic Beanstalk</title><content type='html'>Amazon this week announced <a href="http://aws.amazon.com/elasticbeanstalk/">Elastic Beanstalk</a>, a managed <a href="http://tomcat.apache.org/">Apache Tomcat</a> service for AWS. Naturally, I had to try JRuby on it.<div><br /></div><div>First, the bad:</div><div><ul><li>AWSEB is really slow to deploy stuff. Several times it got "stuck" and I waited for more than 30 minutes for it to recover. It did not appear to be an app issue, since the app came up just fine.</li><li>The default instance size is t1.micro. I was able to get a Rails app to boot there, but it's a very underpowered size.</li><li>It appears to start up JVMs with 256MB of memory max and 64MB of permgen. For a larger app, or one with many Rails instances, that might not be enough. For a "threadsafe" Rails app, though, it's plenty.</li><li>The default EC2 load balancer for the new Beanstalk instance is set to ping the "/" URL. If you don't rig up a / route in your Rails app (like I forgot to do) the app will come up for a few minutes and immediately get taken out.</li></ul><div>And the good news: it works great once you get past the hassles! Here's the process that worked for my app (assuming app is already build and ready for deploy).</div></div><div><br /></div><div>Preparing the app:</div><div><ul><li>Ensure jruby-openssl is in Gemfile. Rails seems to want it in production mode.</li><li>Edit config/environments/production.rb to enable threadsafe mode.</li><li>`warble`</li></ul><div>Preparing Elastic Beanstalk:</div></div><div><ul><li>Create a new instance, specifying the .war file Warbler created above as the app to deploy</li><li>There is no step two</li></ul><div>Once the instance has been prepared, you may want to resize it to something larger than t1.micro if it's meant to be a real app...but it should boot ok.</div></div><div><br /><div><a onblur="try {parent.deselectBloggerImageGracefully();} catch(e) {}" href="http://i.min.us/ieh0Hc.png"><img style="margin:0 10px 10px 0;cursor:pointer; cursor:hand;width: 452px; height: 420px;" src="http://i.min.us/ieh0Hc.png" border="0" alt="" /></a><br /></div><br /></div><div>Have fun!</div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/8236105658793522179/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2011/01/jruby-on-rails-on-amazon-elastic.html#comment-form' title='5 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/8236105658793522179'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/8236105658793522179'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2011/01/jruby-on-rails-on-amazon-elastic.html' title='JRuby on Rails on Amazon Elastic Beanstalk'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>5</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-6160623388426443665</id><published>2011-01-05T16:04:00.000-08:00</published><updated>2011-01-25T21:44:30.202-08:00</updated><title type='text'>Representing Non-Unicode Text on the JVM</title><content type='html'>JRuby is an implementation of Ruby, and in order to achieve the high level of compatibility we boast we've had to put in some extra work. Probably the biggest area is in management of String data.<br /><br /><span style="font-weight:bold;">Strings in Ruby 1.8</span><br /><br />Ruby 1.8 did not differentiate between strings of bytes or strings of characters. A String was just an array of bytes, and the representation of those bytes was dependent on how you used them. You could do regular expression matches against non-text binary sequences (used by some to parse binary formats like PNG). You could treat them as UTF-8 text by setting global encoding variables to assume UTF-8 in literal strings. And you could use the same methods for dealing with strings of bytes you would with strings of characters...split, gsub, and so on. The lack of a character abstraction was painful, but the ability to do character-string operations against byte-strings was frequently useful.<br /><br />In order to support all this, we were forced in JRuby to also represent strings as byte[]. This was not an easy decision. Java's strings are all UTF-16 internally. By moving to byte[]-based strings, we lost many benefits of being on the JVM, like built-in regular expression support, seamless passing of strings to Java methods, and easy interaction with any Java libraries that accept, return, or manipulate Java strings. We eventually had to implement our own regexp engines (or the byte[]-to-char[]-to-byte[] overhead would kill us) and calls from Ruby to Java still pay a cost to pass Strings.<br /><br />But we got a lot out of it too. We would not have been able to represent binary content easily with a char[]-based string, since it would either get garbled (when Java tried to decode it) or we'd have to force the data into only the lower byte of each char, doubling the size of all strings in memory. We have some of the fastest String IO capabilities of any JVM language, since we never have to decode text. And most importantly, we've achieved an incredibly high level of compatibility with C Ruby that would have been difficult or impossible forcing String data into char[].<br /><br />There's also another major benefit: we can support Ruby 1.9's "multilingualization".<br /><br /><span style="font-weight:bold;">Strings in Ruby 1.9</span><br /><br />Ruby 1.9 still represents all string data as byte[] internally, but it adds an additional field to all strings: Encoding (there's also "code range", but it's merely an internal optimization).<br /><br />The biggest problem with encoded text in MRI was that you never knew what a string's encoding was supposed to be; the String object itself only aggregated byte[], and if you ever ended up with mixed encodings in a given app, you'd be in trouble. Rails even introduced its own "Chars" type specifically to deal with the lack of encoding support.<br /><br />In Ruby 1.9, however, Strings know their own encoding. Strings can be forced to a specific encoding or transcoded to another. IO streams are aware of (and configurable for) external and internal encodings, and there's also default external and internal encodings. And you can still deal with raw binary data in the same structure and with the same String-manipulating features. For a full discussion of encoding support in Ruby 1.9, see <a href="http://yehudakatz.com/">Yehuda Katz</a>'s excellent post on <a href="http://yehudakatz.com/2010/05/05/ruby-1-9-encodings-a-primer-and-the-solution-for-rails/">Ruby 1.9 Encodings: A Primer</a>.<br /><br />As part of JRuby 1.6, we've been working on getting much closer to 100% compatible with Ruby 1.9. Of course this has meant working on encoding support. Luckily, we had a hacking machine some years ago in Marcin Mielzynski, who implemented not only our encoding-agnostic regexp engine (a port of Oniguruma from C), but also our byte[]-based String logic and almost all of our Encoding support. The remaining work has trickled in over the subsequent years, leading up the the last few months of heavy activity on 1.9 support.<br /><br /><span style="font-weight:bold;">How It Works</span><br /><br />You might find it interesting to know how all this works, since JRuby is a JVM-based language where Strings are supposed to be UTF-16 always.<br /><br />First off, String, implemented by the <a href="https://github.com/jruby/jruby/blob/master/src/org/jruby/RubyString.java">RubyString</a> class. RubyString aggregates an Encoding and an array of bytes, using a structure we call <a href="https://github.com/jruby/bytelist/blob/master/src/org/jruby/util/ByteList.java">ByteList</a>. ByteList represents an arbitrary array of bytes, a view into them, and an encoding. All operations against a String operate within RubyString's code against ByteLists.<br /><br />IO streams, implemented by <a href="https://github.com/jruby/jruby/blob/master/src/org/jruby/RubyIO.java">RubyIO</a> (and subclasses) and ChannelStream/ChannelDescriptor, accept and return ByteList instances. ByteList is the text/binary currency of JRuby...our java.lang.String.<br /><br />Regexp is implemented in <a href="http://">RubyRegexp</a> using <a href="https://github.com/jruby/joni">Joni</a>, our Java port of the Oniguruma regular expression library. Oniguruma accepts byte arrays and uses encoding-specific information at match time to know what constitutes a character in that byte array. It is the only regular expression engine on the JVM capable of dealing with encodings other than UTF-16.<br /><br />The JRuby parser also ties into encoding, using it on a per-file basis to know how to encode each literal string it encounters. Literal strings in the AST are held in <a href="https://github.com/jruby/jruby/blob/master/src/org/jruby/ast/StrNode.java">StrNode</a>, which aggregates a ByteList and constructs new String instances from it.<br /><br />The compiler is an interesting case. Ideally we would like all literal strings to still go into the class file's constant pool, so that they can be loaded quickly and live as part of the class metadata. In order to do this, the byte[]-based string content is forced into a char[], which is forced into a java.lang.String that goes in the constant pool. At load time, we unpack the bytes and return them to a ByteList that knows their actual encoding. Dare I claim that JRuby is the first JVM language to allow representing literal strings in encodings other than UTF-16?<br /><br /><span style="font-weight:bold;">What It Means For You</span><br /><br />At the end of the day, how are you affected by all this? How is your life improved?<br /><br />If you are a Ruby user, you can count on JRuby having solid support for Ruby 1.9's "M17N" strings. That may not be complete for JRuby 1.6, but we intend to take it as far as possible. JRuby 1.6 *will* have the lion's share of M17N in-place and working.<br /><br />If you are a JVM user...JRuby represents the *only* way you can deal with arbitrarily-encoded text without converting it to UTF-16 Unicode. At a minimum, this means JRuby has the potential to deal with raw wire data much more efficiently than libraries that have to up-convert to UTF-16 and downconvert back to UTF-8. It may also mean encodings without complete representation in Unicode (like Japanese "emoji" characters) can *only* be losslessly processed using JRuby, since forcing them into UTF-16 would either destroy them or mangle characters. And of course no other JVM language provides JRuby's capabilities for using String-like operations against arbitrary binary data. That's gotta be worth something!<br /><br />I want to also take this opportunity to again thank Marcin for his work on JRuby in the past; Tom Enebo for his suffering through encoding-related parser work the past few weeks; Yukihiro "Matz" Matsumoto for adding encoding support to Ruby; and all JRuby committers and contributors who have helped us sort out M17N for JRuby 1.6.</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/6160623388426443665/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2011/01/representing-non-unicode-text-on-jvm.html#comment-form' title='2 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/6160623388426443665'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/6160623388426443665'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2011/01/representing-non-unicode-text-on-jvm.html' title='Representing Non-Unicode Text on the JVM'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>2</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-2769363753545887472</id><published>2011-01-04T03:31:00.000-08:00</published><updated>2011-01-25T21:44:30.216-08:00</updated><title type='text'>Flat and Graph Profiles for JRuby 1.6</title><content type='html'><div>Sometimes it's the little things that make all the difference in the world.</div><br /><div>For a long time, we've extolled the virtues of the amazing JVM tool ecosystem. There's literally dozens of profiling, debugging, and monitoring tools, making JRuby perhaps the best Ruby tool ecosystem you can get. But there's a surprising lack of tools for command-line use, and that's an area many Rubyists take for granted.</div><br /><div>To help improve the situation, we recently got the ruby-debug maintainers to ship our JRuby version, so<a href="http://blog.headius.com/2010/12/jruby-finally-installs-ruby-debug-gem.html"> JRuby has easy-to-use command-line Ruby debugging support</a>. You can simply "gem install ruby-debug" now, so we'll stop shipping it in JRuby 1.6.</div><br /><div>We also shipped a basic "flat" instrumented profiler for JRuby 1.5.6. It's almost shocking how few command-line profiling tools there are available for JVM users; most require you to boot up a GUI and click a bunch of buttons to get any information at all. Even when there are tools for profiling, they're often bad at reporting results for non-Java languages like JRuby. So we decided to whip out a --profile flag that gives you a basic, flat, instrumented profile of your code.</div><br /><div>To continue this trend, we enlisted the help of <a href="http://danlucraft.com/">Dan Lucraft</a> of the <a href="http://redcareditor.com/">RedCar</a> project to expand our profiler to include "graph" profiling results. Dan previously implemented JRuby support for the "ruby-prof" project, a native extension to C Ruby, in the form of "<a href="https://github.com/danlucraft/jruby-prof">jruby-prof</a>" (which you can install and use today on any recent JRuby release). He was a natural to work on the built-in profiling support.</div><br /><div>For the uninitiated, "flat" profiles just show how much time each method body takes, possibly with downstream aggregate times and total aggregate times. This is what you usually get from built-in command-line profilers like the "hprof" profiler that ships with Hotspot/OpenJDK. Here's a "flat" profile for a simple piece of code.</div><br /><div><pre>~/projects/jruby ➔ jruby --profile.flat -e "def foo; 100000.times { (2 ** 200).to_s }; end; foo"<br />Total time: 0.99<br /><br /> total self children calls method<br />----------------------------------------------------------------<br /> 0.99 0.00 0.99 1 Object#foo<br /> 0.99 0.08 0.90 1 Fixnum#times<br /> 0.70 0.70 0.00 100000 Bignum#to_s<br /> 0.21 0.21 0.00 100000 Fixnum#**<br /> 0.00 0.00 0.00 145 Class#inherited<br /> 0.00 0.00 0.00 1 Module#method_added</pre></div><br /><div>A "graph" profile shows the top N call stacks from your application's run, breaking them down by how much time is spent in each method. It gives you a more complete picture of where time is being spent while running your application. Here's a "graph" profile (abbreviated) for the same code.</div><br /><div><pre>~/projects/jruby ➔ jruby --profile.graph -e "def foo; 100000.times { (2 ** 200).to_s }; end; foo"<br />%total %self total self children calls name<br />---------------------------------------------------------------------------------------------------------<br />100% 0% 1.00 0.00 1.00 0 (top)<br /> 1.00 0.00 1.00 1/1 Object#foo<br /> 0.00 0.00 0.00 145/145 Class#inherited<br /> 0.00 0.00 0.00 1/1 Module#method_added<br />---------------------------------------------------------------------------------------------------------<br /> 1.00 0.00 1.00 1/1 (top)<br /> 99% 0% 1.00 0.00 1.00 1 Object#foo<br /> 1.00 0.09 0.91 1/1 Fixnum#times<br />---------------------------------------------------------------------------------------------------------<br /> 1.00 0.09 0.91 1/1 Object#foo<br /> 99% 8% 1.00 0.09 0.91 1 Fixnum#times<br /> 0.70 0.70 0.00 100000/100000 Bignum#to_s<br /> 0.21 0.21 0.00 100000/100000 Fixnum#**<br />---------------------------------------------------------------------------------------------------------<br /> 0.70 0.70 0.00 100000/100000 Fixnum#times<br /> 69% 69% 0.70 0.70 0.00 100000 Bignum#to_s<br />---------------------------------------------------------------------------------------------------------<br /> 0.21 0.21 0.00 100000/100000 Fixnum#times<br /> 21% 21% 0.21 0.21 0.00 100000 Fixnum#**<br />---------------------------------------------------------------------------------------------------------<br /> 0.00 0.00 0.00 145/145 (top)<br /> 0% 0% 0.00 0.00 0.00 145 Class#inherited<br />---------------------------------------------------------------------------------------------------------<br /> 0.00 0.00 0.00 1/1 (top)<br /> 0% 0% 0.00 0.00 0.00 1 Module#method_added</pre></div><br /><div>As you can see, you get a much better picture of why certain methods are taking up time and what component calls are contributing most to that time.</div><br /><div>We haven't settled on the final command-line flags, but look for the new graph profiling (and the cleaned-up flat profile) to ship with JRuby 1.6 (real soon now!)</div></content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/2769363753545887472/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2011/01/flat-and-graph-profiles-for-jruby-16.html#comment-form' title='3 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/2769363753545887472'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/2769363753545887472'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2011/01/flat-and-graph-profiles-for-jruby-16.html' title='Flat and Graph Profiles for JRuby 1.6'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>3</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-8270769560000473811</id><published>2010-12-23T14:01:00.000-08:00</published><updated>2011-01-25T21:44:30.233-08:00</updated><title type='text'>Improved JRuby Startup by Deferring Gem Plugins</title><content type='html'>Another present for you JRubyists out there!<br /><br />JRuby has had notoriously bad startup times. Not as bad as, say, IronRuby (sorry guys!), but definitely a big fat hit every time you need to run some Ruby code from the command line. Some of this overhead was related to JRuby, and we've steadily worked to improve that over the years. Some of it is due to the JVM, most commonly due to running on the "server" Hotspot VM or another JVM that does not have an interpreter (both of which start up considerably slower than Hotspot/OpenJDK's "client" mode). I've blogged <a href="http://blog.headius.com/2010/03/jruby-startup-time-tips.html">tips and tricks for JRuby startup</a> before, and these mostly apply to vanilla JRuby startup performance.<br /><br />However, a large part of the overhead was not specifically due to JRuby or the JVM, but to RubyGems. RubyGems in version 1.3 added support for "plugins", whereby gems could include a specially-named file to extend the functionality of RubyGems itself. Most of these plugins added command-line tools like "gem push" for pushing a new gem to gemcutter.org (now built-in for pushing to rubygems.org). Unfortunately, the feature was originally added by having RubyGems do a full scan of all installed gems on every startup. If you only had a few gems, this was a minor problem. If you had more than a few, it became a big fat O(N) problem, where each of those N could be arbitrarily complex in themselves.<br /><br />The good news is that it looks like my proposed change – <a href="http://rubyforge.org/pipermail/rubygems-developers/2010-December/005898.html">making plugin scanning happen *only* when using the "gem" command</a> – appears likely to be approved for RubyGems 1.4, due out reasonably soon.<br /><br />Here's the <a href="https://gist.github.com/751969">patch</a> and the impact to RubyGems startup times are below. The first two times are without the patch, with the first time against a "cold" filesystem. The final time is with the patch in place. In all cases, it's against my local JRuby working copy, which has around 500 gems installed.<br /><br /><pre>~/projects/jruby ➔ jruby -e "t = Time.now; require 'rubygems'; puts Time.now - t"<br />17.09<br /><br />~/projects/jruby ➔ jruby -e "t = Time.now; require 'rubygems'; puts Time.now - t"<br />6.959<br /><br />~/projects/jruby ➔ git stash pop<br /># On branch master<br /># Changed but not updated:<br /># (use "git add &lt;file&gt;..." to update what will be committed)<br /># (use "git checkout -- &lt;file&gt;..." to discard changes in working directory)<br />#<br /># modified: lib/ruby/site_ruby/1.8/rubygems.rb<br /># modified: lib/ruby/site_ruby/1.8/rubygems/gem_runner.rb<br />...<br /><br />~/projects/jruby ➔ jruby -e "t = Time.now; require 'rubygems'; puts Time.now - t"<br />0.481</pre><br /><br />It's truly a shocking difference, and it's easy to see why JRuby (plus RubyGems) has had such a bad startup-time reputation.<br /><br />I've already made this change locally to JRuby's copy of RubyGems, which should help any users working against JRuby master. The change will almost certainly ship in JRuby 1.6, with RCs showing up in the next couple weeks. So with this change and my <a href="http://blog.headius.com/2010/03/jruby-startup-time-tips.html">JRuby startup tips</a>, we're on the road to a much more pleasant JRuby experience.<br /><br />Happy Hacking!</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/8270769560000473811/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2010/12/improved-jruby-startup-by-deferring-gem.html#comment-form' title='5 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/8270769560000473811'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/8270769560000473811'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2010/12/improved-jruby-startup-by-deferring-gem.html' title='Improved JRuby Startup by Deferring Gem Plugins'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>5</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-518658072475549538</id><published>2010-12-23T13:31:00.000-08:00</published><updated>2011-01-25T21:44:30.245-08:00</updated><title type='text'>JRuby Finally Installs ruby-debug Gem</title><content type='html'>This should be a great Christmas present for many of you.<br /><br />After over three years, the "ruby-debug" gem finally installs properly on JRuby.<br /><br /><pre>~/projects/jruby ➔ gem install ruby-debug<br />Successfully installed ruby-debug-base-0.10.4-java<br />Successfully installed ruby-debug-0.10.4<br />2 gems installed</pre><br /><br />Back in 2007, folks working on NetBeans, Eclipse, and IntelliJ support for Ruby came together to build a new version of the ruby-debug backend that would work on JRuby. They shared the effort, we JRuby guys added features they needed to do a clean port of the ruby-debug C code, and the ultimate result was a ruby-debug-base gem that isolated the platform-specific bits.<br /><br />For whatever reason, the JRuby version of ruby-debug-base never got pushed as a real, live "-java" gem. This meant that you had to download ruby-debug-base-VERSION-java.gem yourself to get ruby-debug to install. To make this process easier, we even shipped ruby-debug and ruby-debug-base preinstalled in JRuby 1.5.<br /><br />Unfortunately, this was only a partial answer. Many libraries and applications want to install all their dependencies clean. If one of those dependencies was ruby-debug, it would fail to install. Rails even includes special JRuby-specific lines in its default Bundler Gemfile to exclude ruby-debug when bundling on JRuby.<br /><br />All that nonsense ends today. Rocky Bernstein, one of the maintainers of the ruby-debug gem, agreed to push our ruby-debug-base to the canonical rubygems.org repository. As a result, ruby-debug now installs properly on JRuby. It only took three years to get that gem pushed (by nobody's fault...I think everyone expected everyone else to follow through on it).<br /><br />Merry Christmas, Happy Chanukah, Joyous Kwanza, and enjoy your holiday season!</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/518658072475549538/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2010/12/jruby-finally-installs-ruby-debug-gem.html#comment-form' title='4 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/518658072475549538'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/518658072475549538'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2010/12/jruby-finally-installs-ruby-debug-gem.html' title='JRuby Finally Installs ruby-debug Gem'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>4</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-445107713335760346</id><published>2010-12-23T11:43:00.000-08:00</published><updated>2011-01-25T21:44:30.258-08:00</updated><title type='text'>Quick Thoughts on Oracle/Apache and the Java TCK</title><content type='html'>This was going to be a reply to a friend on Twitter, when I realized it would be several tweets and I might as well put them in one place.<br /><br />Grant Michaels (@grantmichaels) sent me this link to the <a href="http://jcp.org/aboutJava/communityprocess/summaries/2010/October2010-public-minutes.html">October JCP (Java Community Process) EC (Expert Committee) meeting notes</a>. The Apache/<a href="http://en.wikipedia.org/wiki/Technology_Compatibility_Kit">TCK (Technology Compatibility Kit)</a> issue was discussed at length.<br /><br />For those of you in the dark, Apache recently resigned from the JCP because of the ongoing dispute over their "<a href="http://harmony.apache.org/">Harmony</a>" OSS (Open-Source Software) Java implementation's inability to get an unencumbered license to the Java TCK. Passing the TCK is a requirement for an implementation to officially be accepted as "Java".<br /><br />I had heard about this problem from a distance, but only recently started to understand its complexity. The TCK includes FOU (Field of Use) clauses preventing TCK-tested implementations other than OpenJDK from being released as open-source. Only implementations "largely" based on OpenJDK (Open Java Development Kit, Sun's GPLed "Hotspot" VM and class libraries) are allowed to get around this requirement. Apache's Harmony, being entirely independent and <a href="http://www.apache.org/licenses/LICENSE-2.0.html">Apache-licensed</a>, does not qualify.<br /><br />If that were all there is to it, Apache would not have any real grounds for complaining. But the <a href="http://jcp.org/aboutJava/communityprocess/JSPA2.pdf">JSPA (Java Specification Participation Agreement, PDF)</a> requires only unencumbered specifications, reference implementations, and test kits be submitted to the JCP. This sets the stage for an ugly licensing battle that has stymied Java's progress for over three years.<br /><br />I'm not going to do a tl;dr post on this like I did with <a href="http://blog.headius.com/2010/08/my-thoughts-on-oracle-v-google.html">Oracle/Google</a>. Instead, just a few quick thoughts as I read through the notes.<br /><br /><span style="font-weight:bold;">Overview</span><br /><br />The first thing you will notice is that this isn't a simple cut-and-dried issue. The meeting notes express Oracle's position as outwardly fearful of Harmony leading to many downstream forks, with no recourse for asserting they fulfill the requirements of the Java specification. There seems to be an implied stab at Android here, which uses Harmony's class libraries atop the Dalvik VM to implement a substantial portion (but not all) of what looks like Java SE 5. Oracle states this decision is final; Apache will not be granted an unencumbered TCK license. Oracle is not known for changing their minds.<br /><br />Several EC members, including Doug Lea and Josh Bloch, point out that it's fairly clear the encumbered TCK violates the JSPA's openness clauses. Oracle refuses to comment on this "legal" matter. Doug suggests that EC members might be able to vote to move Java forward with a clearer conscience if the JSPA were amended to make the encumbered TCK "legal".<br /><br />Another point brought up by several members is the frustration that they have to deal with licensing at all. They recall a golden age of the JCP where it actually voted on technical matters rather than arguing over licensing.<br /><br />IBM declares they are unhappy with this decision, but even more unhappy that the Java platform has stagnated because of it for so long. IBM would eventually go on to <a href="http://www.jcp.org/en/jsr/results?id=5111">vote "yes" to the disputed Java 7 JSR</a>, even in the presence of the apparent JSPA violation.<br /><br />Apache representative and longtime Harmony advocate Geir Magnusson also weighed in. He argued that the health of the platform would only be bolstered by allowing for many independent open-source implementations, and damaged by disallowing them. When asking for a clarification of why OpenJDK gets a free pass, Adam Messinger (Oracle) stated that he didn't want to answer a legal question, but that OpenJDK's GPL (Gnu Public License) requires reciprocity from downstream forks, reducing the damage and confusion they might cause if released publicly without full spec compliance (I'm paraphrasing based on the notes here).<br /><br />Toward the end of the licensing discussion, Adam again called for all memory organizations to participate in OpenJDK. It's fairly clear from these notes and from previous announcements and discussions that Oracle intends for OpenJDK to be the "one true OSS Java", and for all comers to contribute to it. They even managed to get Apple and IBM, longtime GPL foes, to join the family. Apache doesn't do GPL, and so Apache will not contribute to or base projects off OpenJDK.<br /><br /><span style="font-weight:bold;">No Right Answer</span><br /><br />Now we enter entirely into the realm of my own opinions.<br /><br />At a glance, I hate Oracle for disallowing free use of the TCK. Nobody should be disallowed from implementing their own OSS Java. Harmony is a promising project that would have a promising future if the cloud of licensing, patent protection, and "compliance" could be lifted. It seems that's nearly impossible now.<br /><br />I also hate to see a good project like Harmony "die" or be stuck in legal limbo. I'm sure dozens of developers have poured their hearts into Harmony, and they deserve to see it thrive.<br /><br />On the other hand, I applaud Oracle for so vigorously promoting OpenJDK. OpenJDK is certainly a more mature JVM and class library than Harmony, having been Sun's official Java implementation for years and years. Bringing IBM and Apple into OpenJDK will help ensure it moves forward on all platforms of note. If you look only at OpenJDK, OSS Java is stronger now than it ever has been.<br /><br />Oracle may have a point with the forking concerns. Because Apache's license is so open that anyone can create and release binary-only forks, it would in theory be possible to speckle the Java landscape with Harmony derivatives that are incompatible in subtle ways. Possible, but perhaps unlikely.<br /><br />Ultimately, I feel sad about the direction Oracle has taken, but I'm bolstered by hopes that OpenJDK is going to really thrive and grow, and as a result "Java" will continue to see widespread use and adoption across a variety of platforms.<br /><br /><span style="font-weight:bold;">Other Ways Out</span><br /><br />Now I go into pure speculation-mode.<br /><br />A key problem with the TCK restriction is the fact that the Java specification provides patent grants only to compliant implementations. In other words, if you don't want an Oracle/Google-esque lawsuit once you have billions in the bank, you need to be compliant with the one true TCK. In order to be compliant with the TCK, you can't release your implementation as open source. The patent grant is obviously intended to ensure that third-party implementations are "really" Java.<br /><br />There's a bit of a chicken-and-egg thing here. What if Harmony could pass the TCK? In theory, they may be at that point right now. Does the ability to pass the TCK but the inability to run it without tainting mean they're Java or not? If an unrelated third-party forked Harmony into "Barmony", acquired a TCK license, and proved that it passed...would that mean Harmony could be considered compliant without ever having run the TCK?<br /><br />What if, as in Android, Harmony simply moved forward without claiming they were compliant? Oracle could eventually club them to death with patent bats. Perhaps such a legal battle would force the legal remifications of the JSPA violation to be addressed in court? Could Oracle survive a legal test of their violation in attempting to back up a patent suit?<br /><br /><span style="font-weight:bold;">What It Means For You</span><br /><br />The ultimate question is how this affects Java developers today and going forward.<br /><br />By most estimates, 99% of the worlds Java developers run on one of the standard Java implementations from Oracle, IBM, and other licensees. A massive number of them run atop Hotspot, either in an old closed-source Java 5/6 form or in an OpenJDK 6/7 form. "Nobody" runs Harmony, and so few if any day-to-day Java developers will be affected. That doesn't excuse the situation, but it does soften the actual damage caused.<br /><br />Android is another peculiar case. It would be difficult for it to be tested compliant, even if there weren't FOU restrictions on doing so. It uses Harmony libraries but Dalvik VM. It is also a massive force in mobile development now, and killing it would likely put the final nail in mobile Java's coffin. Oracle has to know this. Oracle also has to know that killing Android would hand the mobile keys over to Apple and Microsoft forever. Could Android switch to using OpenJDK-based class libraries? Would that qualify it as being "largely" based on OpenJDK (noting that the class libraries are the vast majority of the code in OpenJDK)? Oracle/Google is likely to be stuck in court for a long time, while Android continues to expand into televisions and tablets along with telephones.<br /><br />How about projects that build atop Java, like JRuby? Perhaps even higher a percentage of JRuby users are already running atop OpenJDK or an OpenJDK derivative like IcedTea or SoyLatte. Oracle pushing OpenJDK will only benefit those users.<a href="http://ruboto.org/"> Ruboto (JRuby on Android)</a> will follow whatever path Android itself ends up following, and nobody can see that future yet...but it seems unlikely Ruboto will ever die since it's unlikely Android will ever die.<br /><br />In closing...I encourage everyone to read the EC notes and gather as much information as they can before claiming this is now the final "death" of Java. I also encourage everyone to contribute thoughts, clarifications, and speculation in the comments here.<br /><br />Have a happy holiday!</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/445107713335760346/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2010/12/quick-thoughts-on-oracleapache-and-java.html#comment-form' title='5 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/445107713335760346'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/445107713335760346'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2010/12/quick-thoughts-on-oracleapache-and-java.html' title='Quick Thoughts on Oracle/Apache and the Java TCK'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>5</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-4895478538024829801</id><published>2010-09-23T01:39:00.000-07:00</published><updated>2011-01-25T21:44:30.273-08:00</updated><title type='text'>Predator and Prey</title><content type='html'>(This is a repost of an article I wrote in 2004, which I stumbled upon this evening and thought worthy of a reprint. Feel free to rip it up and offer your own commentary. I think it is still 100% valid.)<br /><br />I came up with the most compelling idea for a Disney-style film the other day. (Ok, perhaps not the most compelling idea, but certainly a fair shot at one)<br /><br />Over the years I've heard a number of biologists (ecologists, environmentalists, what have you) comment on (as in expound endlessly upon) something called the "Bambi Syndrome." Simply put, the "Bambi Syndrome" is brought about by cutesy, utopian images of nature, where only unexpected, amorphous entities (usually accompanied by menacing percussion or something equally non-musical) can embody "evil"; it is a view that, in all its splendor and glory, "nature" is "good," while "man" is "bad." The parallel between this viewpoint and several (all?) nature-based Disney films is apparent (although it should be said that Disney is far from being the only perpetrator of "Bambiism").<br /><br />So then, you ask, if nature isn't "good", then what is it? Evil and good are purely human constructions. Truth be told, nothing that exists is innately "good" or "evil". These concepts exist only in the eye of the beholder: to the prey, the successful predator is evil; to the predator, the successful prey is evil.<br /><br />It could then be considered a great disservice to continue teaching these false ideals to our children, no? This has been my opinion, and I have tried to take an approach with my own son of presenting these facts of nature in as unbiased a way as possible--whence springs the compelling idea.<br /><br />Take a typical Disney movie; its clear definition of "good" and "evil" and its even clearer illustration of which roles fall into which category. This movie would begin the same. Also typically, it would be based in nature, perhaps at a very low stratus of the animal kingdom. Predator and prey would be represented by species A (the "good" prey) and B (the "evil" predator). A typical scene ensues, a contest between good and evil, predator and prey. The predator's evil nature is clearly illustrated here, but atypically, the predator wins.<br /><br />Just as people in the audience are questioning their faith in Hollywood, we move up one stratum. The evil predator, returning home with the spoils of war, becomes a gentle, caring mother. She was not simply an "evil" aggressor, bent on death and destruction, but a doting, protective mother, expending her own effort, at risk of her life, to care for her childen. In this way, stratum after stratum, "evil" becomes "good", and the elaborate network that makes up our natural system becomes more recognizable for the purity, neutrality, simplicity of its form.<br /><br />Finally, as you would expect in such a movie, we would arrive at the most prolific of the Great Apes: man. Illustrating that all kingdoms on earth are becoming man's prey, with as much tree-hugging, granola-chomping tripe as possible to make sure we, the lords of creation, masters of destiny, killers of all, Shiva to nature's Brahma , are shown--incontrovertably--as the only pure "evil" on earth, the movie careens ever faster toward some measure of certainty: "Ahh, now I understand the film's message."<br /><br />But man is just another spoke in the wheel. We can easily flip the coin, showing mothers feeding, defending children, innocents preyed upon by murderers, hunters taking prey not for food, but for the feeding of other hungers. We do what we do not out of pure evil, but because it is our capacity to do so to further our own species, further our goals, perpetuate. But we also have a capacity no other species possesses: the ability to create our own destinies. The only true evil we encounter in a world where we nearly reign supreme is ourselves. We daily pit our most animal desires--acqusition of resources and destruction of usurpers--against our knowledge that such desires run rampant will complicate our path through history, perhaps even terminating it. Can such a machine be affected by the changing opinions of a few small components? That is the question we leave for the viewers.<br /><br />The challenge in such a film would almost certainly be not overplaying the hand. No evil must ever appear to be of any different motivation than its antithesis; and man must, in the end, appear as the most schizophrenic creature on Earth. Our "evil" predatory instincts must be tempered by the "good" effects of our fear of intimate and ultimate mortality for us to continue indefinitely. In this, man has another trait not found among the animals: Our system balances on our own decisions alone. With the capacity we will soon possess to control nature completely, without fear of predators, we can only undo ourselves. The balance comes from within.<br /><br />Where will the viewer lie?<br /><br />I'd hope every kid was as confused as possible by then; and eventually a bit more suspicious of being told what is "good" or "evil".</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/4895478538024829801/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2010/09/predator-and-prey.html#comment-form' title='7 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/4895478538024829801'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/4895478538024829801'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2010/09/predator-and-prey.html' title='Predator and Prey'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>7</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-4344974410425965514</id><published>2010-08-15T18:53:00.000-07:00</published><updated>2011-01-25T21:44:30.290-08:00</updated><title type='text'>My Thoughts on Oracle v Google</title><content type='html'>As you've probably heard by now, Oracle has decided to file suit against Google, claiming multiple counts of infringement against Java or JVM patents and copyrights they acquired when they assimilated Sun Microsystems this past year. Since I'm unlikely to keep my mouth shut about even trivial matters, something this big obviously requires at least a couple thousand words.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:x-large;">Who Am I?</span></span><br /><br />Any post of this nature really requires an author to identify where they stand, so their unavoidable biases can be taken with the appropriate dosage of salt. Rather than having you dig through my past and learn who and what I am, I'll just lay it out here.<br /><br />I am a Java developer. I've been a Java developer since 1996 or so, when I got my first University job writing stupid little applets in this new-fangled web language. That job expanded into a web development position, also using Java, and culminated with me joining a few senior developers for a 6-month shared development gig with IBM's then-nascent Pacific Development Center in Vancouver, BC. Since then I've had a string of Java-related jobs...some as a trenches developer, some as a lead, some as "architect", but all of them heavily wrapped up in this thing called Java. And I can't say that I've ever been particularly annoyed with Java as a language or a platform. Perhaps I haven't spent enough time on other runtimes, or perhaps I've got tunnel-vision after being a Java developer for so many years. But I'd like to think that I've become "seasoned" enough as a developer to realize no platform is perfect, and the manifold benefits of the JVM and the Java platform vastly outweigh the troublesome aspects.<br /><br />I am an open-source developer. In the late 90s, I worked in earnest on my first open-source project: the LiteStep desktop replacement for Windows. At the time, the LiteStep project was a loosely-confederated glob of C code and amateur C hackers. Being a Windows user at the time, I was looking to improve my situation...specifically, I had worked for years on a small application called Hack-It that exposed aspects of the win32 API normally unavailable through standard Windows UI elements, and I was interested in taking that further. LiteStep was not my creation. It had many developers before me and many after, but my small contribution to the project was an almost complete rewrite in amateur-friendly C++ and a decoupling of the core LiteStep "kernel" from the various plugin mechanisms. I was also interviewed for a Wired article on the then-new domain of "skinning" computers, desktops, applications, and so on, though none of my quotes made it into the article. After LiteStep, I fell back into mostly anonymous corporate software development, all still using Java and many open-source technologies, but not much of a visible presence in the OSS world. Then, in 2004 while working as the lead "Java EE Architect" for a multi-million-dollar US government contract, I found JRuby.<br /><br />I am a JRuby developer. Since 2004 (or really since late 2005, when I started helping out in earnest) I've been partially responsible for turning JRuby from an interesting novelty project into one of the top Ruby implementations. We've become well known as one of the best-performing – if not the best-performing – Ruby implementations, even faced with increasing competition from the young upstarts. We're also increasingly popular (and perhaps the easiest path) for bringing Ruby and its many paradigm-shifting libraries and frameworks (like Rails) to Java and JVM users around the world – without them having to change platforms or leave any of their legacy code behind. Part of my interest in JRuby has been to bring Ruby to the JVM, plain and simple. I like Ruby, I like the Ruby community, and on most days I like the cockiness and enthusiasm of those community members toward trying crazy new things. But another large part of my interest in JRuby is more sinister: I want to prove to naysayers what a great platform the JVM actually is, and perhaps make them think twice about knee-jerk biases they've carried and cultivated for so many years.<br /><br />You'll notice I refer to JRuby not as "it" or "she" or "he", but as "we". "We've become well known...We're also increasingly popular..." That's not an accident. There's now over five years of my efforts in JRuby, and I consider it to be as much a part of me as I am a part of it. And so because of that, I have a much deeper, emotional investment in the platform upon which JRuby rests.<br /><br />I am a Java developer. I am an open-source developer. I am a JRuby developer and a Ruby fan.<br /><br />I am not a lawyer.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:x-large;">The Facts, According to Me</span></span><br /><br />These are the facts as I see them. You're free to disagree with my interpretation of the world, and I encourage you to do so in the comments, on other forums, over email, or to my face (but buy me a beer first).<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">On Java</span></span><br /><br />The Java platform is big. Really big. You just won't believe how vastly hugely mindbogglingly big it is. And by big, I mean it's everywhere.<br /><br />There are three mainstream JVMs people know about: JRockit (WebLogic's first and then Oracle's after it acquired them), Hotspot (Which came to Sun through an acquisition and eventually became OpenJDK), and J9 (IBM's own JVM, fully-licensed and with all its shots). Upon those three JVMs lives a gigantic world. If you want the details, there's numerous studies and reports about the use of Java in all manner of business, from the hippest new startups (Twitter recently switched much of their stack to the JVM) to the oldest of the old financial concerns. It's the favored choice for government server applications, the strongest not-quite-completely-Free managed runtime for open-source libraries and applications, and now with Android it's rapidly becoming one of the strongest (if not the strongest) mobile OS platform (even though Android isn't *really* Java, as I'll get into later). You may love or hate Java, but I guarantee it's part of your life in some way or another.<br /><br />There are a few open-source implementations of Java. The most well-known is OpenJDK, the Hotspot JVM that Sun relicensed under the GPL and set Free into the world. There's also Apache Harmony, whose class libraries form part of Dalvik's (Android's VM) Java-compatibility layer. There's GNU Classpath, a GPL-based implementation of the Java class libraries used for the ahead-of-time Java compiler GCJ. There's JamVM, which leverages Classpath to provide a very light, very minimal, (and very simple) JVM implementation. And there's others of varying qualities and relevance like IKVM (Java for .NET), VMKit (a Java compiler atop LLVM), and so on. OpenJDK is certainly the big daddy, though, and its release as GPL guarantees we'll at least have a solid Java 6 implementation forever.<br /><br />Java is not an entirely open platform, what with the now-obvious encumbrances of patents and copyrights (not to mention draconian policies toward Java's various specifications, which are often very slow to evolve due to the JCP quagmire). That's not a great state of affairs, and if nothing else you have to recognize that folks at Sun at least tried to release the platform from its shackles by chasing OpenJDK. But the process of "freeing" Java has been pretty rocky; OpenJDK itself took years to gain acceptance from OSS purists, and the choice of the GPL has meant that folks afeared of the GPL's "viral" side still had to look for other options (which is a large part of why Apache Harmony was used as part of the basis for Android). Perhaps the biggest nail in the coffin is that Sun's Java test kit, the gold standard of whether an implementation is "compliant" or not, has never been released in open-source form, ultimately binding the hands of developers who wished to build a fully-compatible open-source Java.<br /><br />Java is not an entirely closed platform, either. OpenJDK was a huge step in the direction of Freeing Java, and the Java community in general has a very strong OSS ethos. There's no piece of Java software that isn't at least partially based on open-source componenents, and most Java library, framework, or application developers either initially or eventually open-source some or all of their works. Open-source development and the Java platform go hand-in-hand, and without that relationship the platform would not be where it is today. Contrast that to other popular environments like Microsoft's .NET – which has been admirably Freed through open standards, but which has not yet become synonymous with or popular for OSS development – or Apple's various platforms – which aren't based on open-standards *or* open-source, but which have managed to become many OSS developers' environment of choice...for writing or consuming non-Apple open-source software. Among the corporation-controlled runtimes, the Java platform has more OSS in its blood than all others combined...many times more.<br /><br />Java is not perfect, but it's pretty darn good. Every platform has its warts. The Java platform represents a decade and a half of tradeoffs, and it's impossible in that amount of time to make everyone happy all the time. One of the big contentious items is the addition in Java 5 of parametric polymorphism as a compile-time trick without also adding VM-level support for reifying per-type specializations as .NET can do. But ask most Java developers if they'd rather have nothing at all, and you'll get mixed responses. The sad, crippled version of generics in Java 5 doesn't do everything static-typing purists want, nor does it really extend to runtime at all (making reflective introspection almost impossible), but they do provide some nice surface-level sugar for Java developers. The same can be said of many Java "features" and tradeoffs. JavaEE became an abortively complicated jumble of mistakes (tradeoffs that went bad), but even upstarts that arguably made better decisions initially have themselves graduated into chaos (I believe the Spring framework has now grown even larger than the largest Java EE conglomerate, and Microsoft's periodically reboots their blessed-framework-of-the-week, resulting in an even more disruptive environment than a slow-moving, bulky standard like JavaEE). Designing good software is hard. Designing good *big* software is exponentially harder. Designing good *big* software that pleases everyone is impossible.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">Why People Hate Java</span></span><br /><br />Java is second only to Microsoft's platforms for being both wildly successful and almost universally hated by the self-sure software elite. The reasons for this are manifold and complex.<br /><br />First of all, back in the 90s Java started getting shoved down everyone's throat. Developers were increasingly told to investigate this new platform, since their managers and long-disconnected tech leads kept hearing how great it was from Sun Microsystems, then a big deal in server applications and hardware. So developers that were happily using other environments (many of which exist to this day) often found themselves forced to suck it up and become Java developers. Making matters worse, Java itself was designed to be a fairly limited language...or at least limited in how easily a developer could paint themselves into a corner. Many features those reluctant developers had become used to in other environments were explicitly rejected for Java on the grounds that they added too much complexity, too much confusion, and too little value to trenches developers. So people that were happily doing Perl or C++ or Smalltalk or what have you were suddenly forced into a little J-shaped box and forced to write all those same applications upon Java and the JVM at a time when both were still poorly-suited to those domains. Those folks have had a white-hot hate for anything relating to Java ever since, and many will stop at nothing to see the entire platform ejected into space.<br /><br />Second, as mentioned quickly above, Java in the 90s was simply not that great a platform. It had most of the current warts (classpath issues, VM limitations, poor system-level integration, a very limited language) on top of the fact that it was slow (optimizing JVMs didn't come around until the 2000s), marketed for highly-visible, highly-fickle application domains like desktop and browser-based applications (everyone's cursed a Java app or applet at some point in their life), and still largely driven and controlled by a single company (at a time when many developers were trying to get out from under Microsoft's thumb). It wasn't until Java 1.2 that we started to get a large and diverse update to Java's core libraries. Java 1.3 was the first release to ship Hotspot, which started to get the performance monkey off our backs. Java 1.5 brought the first major changes to the Java language, all designed to aid developers in expressing what they meant in standard ways (like using type-safe enums instead of static final ints, or generics for compiler-level assurances of collection homogeneity). And Java 6, the last major version, made great strides in improving startup time, overall performance, and manageability of JVM processes. Java 7, should it ever ship, will bring new changes to the Java language like support for closures and other syntactic sugar, better system-level integration features as found in NIO.2, and the feather in the cap: VM-level support for function objects and non-standard invocation sequences via Method Handles and InvokeDynamic. But unless you've been a Java developer for the past decade, all you remember is the roaring 90s and the pain Java caused you as a developer or a user.<br /><br />Third, the Java language and environment has stagnated. Given years of declining fortunes at Sun Microsystems, disagreement among JCP members about the direction the platform should go, and a year of uncertainty triggered by Sun's collapse and rescue at the hands of Oracle, it's surprising anything's managed to get done at all. Java 7 is now many years overdue; they were talking about it when I joined Sun in 2006, and hoped to have preview releases within a year. For both technical and political reasons, it's taken a long time to bring the platform to the next level, and as a result many of the truly excellent improvements have remained on the shelf (much to my dismay...we really could use them in JRuby). For fast-moving technology hipsters, that's as good as dying on the vine; you need to shift paradigms on a regular schedule or you're yesterday's news.<br /><br /><span style="font-weight:bold;"><span style="font-style:italic;">Update:</span></span> At least one commenter also pointed out that it took a long time for Java to be "everywhere", and even today most users still need to download and install it at least once on any newly-installed OS. Notable exceptions include Mac OS X, which ships a cracker-jack Java 6 based on Hotspot, and some flavors of Linux that come with some sort of Java installed out of the box. But this was definitely a very real problem; developers were being pushed to write apps and applets in Java, and users were forced to download a multi-megabyte installer just to run them...at a time when downloading multi-megabyte software was often a very painful ordeal. That would put a bad taste in anyone's mouth.<br /><br />It's because of these and similar reasons that folks like Google finally said "enough is enough," and opted to start doing their own things. On the JRuby project, we've routinely hacked around the limitations of the JVM, be they related to its piss-poor process management APIs, its cumbersome support for binding native libraries, or its stubborn reluctance to become the world's greatest dynamic language VM. I've thought on numerous occasions how awesome it would be to spin off a company that took OpenJDK and made it "right" for the kinds of development people want to do today (and I'd love to be a part of that company), but such ventures are both expensive and light on profitability. Nobody pays for platforms or runtimes...they pay for services around those platforms or runtimes, services which are often anathema to the developers of those platforms and runtimes. So it required someone "bigger" to make that happen...someone who could write off the costs of the platform by funding it in creative new ways. Someone with a massive existing investment in Java. Someone with deep pockets and an army of the best developers in the business who love nothing more than a challenge. Someone like Google.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">Why Android?</span></span><br /><br />(Note that a lot of this is based on what information I've managed to glean from various conversations. Clarifications or corrections are welcome.)<br /><br />There's an incredibly successful mobile Java platform out there. One that boasts millions of devices from almost all the major manufacturers, in form factors ranging from crappy mid-00s clamshells to high-end smartphones. A platform with hundreds or thousands of games and applications and freely-available development tools. That platform is Java ME.<br /><br />Java ME started out as an effort to bring Java back to its original roots: as a language and environment for writing embedded applications. The baseline ME profiles are pretty bare; I did some CLDC development years ago and had to implement my own buffered streams and various data structures just to get by. Even the biggest profiles are still fairly restricted, and I don't believe any of them have ever graduated beyond Java 1.3-level featuresets. So Sun did a great job of getting Java ME on devices, back when people cared about Sun...and then they let mobile Java stagnate to a terrible degree while they spent all resources trying to get people to use Java EE and trying to get Java EE to suck less. So while resources were getting poured into EE, people started to form the same opinions of mobile Java they had formed about desktop and server Java years earlier.<br /><br />At the same time, Java ME was one of the few Java-related technologies that brought in money. You see, in order for handset manufacturers to ship (and boast about) Java ME support, they had to license the technology from Sun. It wasn't a huge cash cow, but it was a cow nonetheless. Java ME actually made money for Sun. So in true Sun form, they loused it up terribly.<br /><br />Fast forward to a few years ago. Google, recognizing that mobile devices finally were becoming the next great technology market, decided that leaving the mobile world in the hands of proprietary platforms was a bad idea. Java ME seemed like it could be an answer, but Sun was starting to get desperate for both revenue and relevance...and they'd started to back a completely new horse-that-would-be-cow called JavaFX, which they hoped to pimp as the next great development environment for in-browser and on-device apps alike. They weren't interested in making Java ME be what Google wanted it to be.<br /><br />Google decided to take the hard route: they'd fund development of a new platform, building it entirely from open-source components, and leveraging two of the best platform technologies available: Linux, for the kernel, and Java, for the runtime environment. However there was a problem with Java: it was encumbered by all sorts of patents and copyrights and specifications and restrictions. Hell, even OpenJDK itself, the most complete and competitive OSS implementation of Java, could not be customized and shipped in binary-only form by hardware manufacturers and service providers due to it being GPL. So the answer was to build a new VM, use unencumbered versions of the core Java class libraries, and basically remake the world in a new, copyright and patent-free image. Android was born.<br /><br />There's many parts to Android, several of which I'm not really qualified to talk about. But the application environment that runs atop the Dalvik VM needs some explanation.<br /><br />First, there's the VM. Dalvik is *not* a JVM. It doesn't run JVM bytecode, and you can't ship JVM bytecode expecting it to work on Dalvik. You must recompile it to Dalvik's own bytecode using one of the provided translation tools. This is similar to how IKVM gets Java code to run on .NET: you're not actually running a JVM, you're transforming your code into a different form so it will run on someone else's VM. So it bears repeating, lest anyone get confused: Dalvik is not a JVM...it just plays one on TV.<br /><br />Second, there's the core Java class libraries. Android supports a rough (but large) subset of the Java 1.5 class libraries. That subset is large enough that projects as complicated as JRuby can basically run unmodified on Android, with very few restrictions (a notable one is the fact that since we can't generate JVM bytecode, we can't reoptimize Ruby code at runtime right now). In order to do this without licensing Sun's class libraries (as most other mainstream Java runtimes like JRockit and J9 do), Google opted to go with the not-quite-complete-but-pretty-close Apache Harmony class libraries, which had for years been developed independent of Sun or OpenJDK but never really tested against the Java compatibility kits (and there's a long and storied history behind this situation).<br /><br />So by building their own non-JVM VM and using translated versions of non-Sun, non-encumbered class libraries, Google hoped to avoid (or at least blunt) the possibility that their "unofficial", "unlicensed" mobile Java platform might face a legal test. In short, they hoped to build the open mobile Java platform developers wanted without the legal and financial encumbrances of Java ME.<br /><br />At first, they seemed to be on a gravy train with biscuit wheels.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">Splitting Up the Pie</span></span><br /><br />Sun Microsystems was not amused. A little over a year ago, when several Sun developers started to take an eager interest in Android, we were all told to back off. It wasn't yet clear whether Android stood on solid legal ground, and Sun execs didn't want egg on their face if a bunch of their own employees turned out to be supporting a platform they'd eventually have to attack. Furthermore, it was an embarrassment to see Android drawing in the same developers Sun really, really wanted to look at JavaFX or PersonalJava or whatever the latest attempt to bring developers back might be. Android actually *was* a great platform that supported existing Java developers and libraries incredibly well (without actually being a Java environment), and for the first time there was a serious contender to "standard" Java that Sun had absolutely no control over.<br /><br />To make matters worse, handset manufacturers started to sign on in droves to this new non-Java ME platform, which meant all that technology licensing revenue was reaching a dead end. Nobody (including me) wanted to do Java ME development anymore. Everyone wanted to do Android development.<br /><br />Now we must say one thing to Sun's credit: they didn't do what Oracle is now attempting to do. As James Gosling blogged recently, patent litigation just wasn't in Sun's blood...even if there might have been legal ground to file suit. So while we Sun employees were still quietly discouraged from looking at or talking about Android, the rest of the world took Sun's silence as carte blanche to stuff Android into everything from phones to TVs, and mobile app developers started to think there might be hope for a real competitor to Apple's iPhone. Things might have proceeded in this way indefinitely, with Android continuing to grab market share (it recently passed iPhone in raw numbers with no slowing in sight) and mindshare (Android is far more approachable than almost any other mobile development environment, especially if you're one of the millions of developers who know Java.)<br /><br />And then it all started to go wrong.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">The Mantle of Java Passes to Oracle</span></span><br /><br />If for nothing else, Jonathan Schwartz will be remembered as the man who broke open the Sun piñata, simultaneously releasing more open-source software than any company in history and killing Sun in the process. Either Jonathan had no "step 2" or the inertia of a company built on closed-source products was too great to overcome. In either case, by spring of 2009 Sun was hemorrhaging. Many reports claim that Jonathan had started shopping Sun around to possible buyers as early as 2008, but it wasn't until 2009 that the first candidates started lining up. Initially, it was IBM, hoping to gobble up its former competitor along with the IP, patents, and copyrights they carried. That deal ultimately went south when Sun refused to consider any deal that IBM wouldn't promise to carry to completion, even in the face of regulatory roadblocks sure to come up. Many of us breathed a sigh of relief; if there's any Java company even more firmly stuck in the old world than Sun, it's IBM...and we weren't looking forward to dealing with that.<br /><br />Once that deal fell through, folks like me became resigned to the fact that Sun was nearing the end of its independent life. Years of platform negligence, management incompetence, and resting on laurels had dug a hole far too deep for anyone to climb out of. Would it be Cisco, who had recently started building up an interesting new portfolio of application server hardware and virtualization software? What about VMWare, who had recently gobbled up Springsource and seemed to be making all the right moves toward a large-scale virtualized "everything cloud." Or perhaps Oracle, a long-time partner to Sun, whose software was either Java-based or widely deployed on Sun hardware and operating systems. Dear god, please don't let it be Oracle.<br /><br />Don't get me wrong...Oracle's a highly successful company. They've managed to turn almost every acquisition into gold while coaxing profitability out of just about every one of their divisions. But Oracle's not a developer-oriented company (like Sun)...it's a profit-oriented company (unlike Sun, sadly), and you need to either feed the bottom line or feed others in the company that do. So when it turned out that Oracle would gobble up Sun, many of us OSS folks started to get a little nervous.<br /><br />You see, many of us at Sun had been actively trying to change the perception of the platform from that of a corporate, enterprisey, closed world to that of a great VM with a great OSS ecosystem and an open-source reference implementation. Folks like Jonathan believed that by freeing Java we'd free the platform, and both the platform and the developer community would be better for it. We were half right...the OpenJDK genie is out of the bottle, and there's basically no way to put it back now (and for that, the world owes Sun a great debt). But only part of the platform was Freed...the patents and copyrights surrounding Hotspot and Java itself remained in place, carefully tucked away in the vault of a company that just didn't mount patent or copyright-driven legal attacks.<br /><br />Oracle, now in control of those patents and copyrights, obviously has different plans.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:x-large;">The Suit</span></span><br /><br />So now, after spending 4000 words of your time, we come to the meat of the article: the actual Oracle v Google suit. The full text is provided various places online, though the <a href="http://en.swpat.org/wiki/Oracle_v._Google_(2010,_USA)">Software Patents Wiki</a> has probably the best collection of related facts (though the wiki-driven discussions of the actual patents are woefully inaccurate).<br /><br />The suit largely comes down to a patent-infringement battle. Oracle claims that by developing and distributing Android, Google is in violation of seven patents. There's also an amorphous copyright claim without much backing information ("Google probably stole something copyrighted so we'll list a bunch of stuff commonly stolen in that way"), so we'll skip that one today.<br /><br />Before looking at the actual patents involved, I want to make one thing absolutely clear: Oracle has not already won this suit. Even after a couple days of analysis, nobody has any idea whether they *can* win such a suit, given that Google seems to have taken great pains to avoid legal entanglements when designing Android. So everybody needs to take a deep breath and let things progress as they should, and either trust that things will go the right direction or start doing your damndest to make sure they go the right direction.<br /><br />With that said, let's take a peek at the patents, one by one. And as always, the "facts" here are based on my reading of the patents and my understanding of the related systems.<br /><br /><span style="font-weight:bold;"><a href="http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&amp;Sect2=HITOFF&amp;p=1&amp;u=/netahtml/PTO/search-bool.html&amp;r=1&amp;f=G&amp;l=50&amp;co1=AND&amp;d=PTXT&amp;s1=6,125,447.PN.&amp;OS=PN/6,125,447&amp;RS=PN/6,125,447"><span class="Apple-style-span" style="font-size:large;">Protection Domains To Provide Security In A Computer System (6,125,447)</span></a><span class="Apple-style-span" style="font-size:large;"> and </span><a href="http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&amp;Sect2=HITOFF&amp;p=1&amp;u=/netahtml/PTO/search-bool.html&amp;r=1&amp;f=G&amp;l=50&amp;co1=AND&amp;d=PTXT&amp;s1=6,192,476.PN.&amp;OS=PN/6,192,476&amp;RS=PN/6,192,476"><span class="Apple-style-span" style="font-size:large;">Controlling Access To A Resource (6,192,476)</span></a></span><br /><br />The first two patents describe the Java Security Policy system, for controlling access to resources. One of the least-interesting but most-important aspects of the Java platform is its approach to security. Code under a specific classloader or thread can be forced to comply with a specific security policy by installing a security manager. These permissions control just about every aspect of the code's interaction with the JVM and with the host operating system: loading new code, reflectively accessing existing classes, accessing system-level resources like devices and filesystems, and so on. It's even easy for you to build up security policies of your own by checking for custom-named permissions and only granting them when appropriate. It's a pretty good system, and one of the reasons Java has a much stronger security track record than other runtimes that don't have pervasive security in mind from the beginning.<br /><br />In order to host applications written for the Java platform, and to sandbox them in a compatible way, Android necessarily had to support the same security mechanisms. The problem here is the same problem that plagues many patents: what boils down to a fairly simple and obvious way to solve a problem (associate pieces of code with sets of permissions, don't let that code do anything outside those permissions) becomes so far-reaching that almost any reasonable *implementation* of that idea would violate these patents. In this case the '447 and '476 patents do describe mechanisms for implementing Java security policies, but even that simple implementation is very vague and would be hard to avoid with even a clean-room implementation.<br /><br />Now I do not know exactly how Android implements security policies, but it's probably pretty close to what's described in these patents...since just about every implementation of security policies would be pretty close to what's described.<br /><br /><span style="font-weight:bold;"><a href="http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&amp;Sect2=HITOFF&amp;p=1&amp;u=/netahtml/PTO/search-bool.html&amp;r=1&amp;f=G&amp;l=50&amp;co1=AND&amp;d=PTXT&amp;s1=5,966,702.PN.&amp;OS=PN/5,966,702&amp;RS=PN/5,966,702"><span class="Apple-style-span" style="font-size:large;">Method And Apparatus For Preprocessing And Packaging Class Files (5,966,702)</span></a></span><br /><br />This is basically the patent governing the "Pack200" compression format provided as part of the JDK and used to better-compress class file archives.<br /><br /><span style="font-weight:bold;"><span style="font-style:italic;">Update:</span></span> Alex Blewitt has posted a <a href="http://alblue.bandlem.com/2010/08/more-details-on-5966702-and-pack200.html">discussion of the Pack200 specification</a>. He says this patent isn't nearly as comprehensive, but that it may touch upon how Pack200 works. His post is a more complete treatment of the details of the class file format and how Pack200 improves compression ratios for class archives. It also occurs to me now that this patent could be related to mobile/embedded Java too, where better compression would obviously have an enormous savings.<br /><br />Java class files are filled with redundant data. For example, every class that contains code that calls PrintStream.println (as in System.out.println) contains the same "constant pool" entry identifying that method by name, a la "java/io/PrintStream.println:(Ljava/lang/String;)V". Every field lookup, class reference, literal string, or method invocation will have some sort of entry in the constant pool. Pack200 takes advantage of this fact by compressing all class files as a single unit, batching duplicate data into one place so that the actual unique class data boils down to just the unique class, method, and code structure.<br /><br />The reason for having a separate compression format is because "zip" files, which includes Java's "jar" files, are notoriously bad at compressing many small files with redundant data. Because one of the features of the "zip" format is that you can easily pull a single file out, compressing all files together as a single unit prevents introducing any interdependencies between those files or a global table. This is a large part of why compression formats like "tar.gz" do a better job of compressing many small files: tar turns many files into one file, and gzip or bzip2 compress that one large file as a single unit (conversely, this is why you can't easily get a single file out of a tarball).<br /><br />On Android, this is accomplished in a similar way by the "dex" tool, which in the process of translating JVM bytecode into Dalvik bytecode also localizes all duplicate class data in a single place. The general technique is standard data compression theory, so presumably the novelty lies in applying decades-old compression theory specifically to Java classfile structure.<br /><br />If I've lost you at this point, we can summarize it this way: part of Oracle's suit lies in a patent for a better compression mechanism for archives containing many class files that takes advantage of redundant data in those files.<br /><br />Are you laughing yet?<br /><br /><span style="font-weight:bold;"><a href="http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&amp;Sect2=HITOFF&amp;p=1&amp;u=/netahtml/PTO/search-bool.html&amp;r=1&amp;f=G&amp;l=50&amp;co1=AND&amp;d=PTXT&amp;s1=7,426,720.PN.&amp;OS=PN/7,426,720&amp;RS=PN/7,426,720"><span class="Apple-style-span" style="font-size:large;">System And Method For Dynamic Preloading Of Classes Through Memory Space Cloning Of A Master Runtime System Process (7,426,720)</span></a></span><br /><br />I'm not sure this patent ever saw the light of day in a mainstream JVM implementation. It describes a mechanism by which a master parent process could pre-load and pre-initialize code for a managed system, and then new processes that need to boot quickly would basically be memory-copied (plus copy-on-write friendly) "forks" of that master process, with the master maintaining overall control of those child processes through some sort of IPC.<br /><br />Ignore for the moment the obvious prior art of "fork" itself as applied to pre-initializing application state for many children. Anyone who's ever used fork to initialize a heavy process or runtime to avoid the cost of reinitializing children has either violated this patent (if done since 2003) or has a compelling case for prior art (if done before 2003).<br /><br />It's likely that this patent was formulated as an answer to the poor semantics of running many applications under the same JVM. Java servlets and later Java EE made it possible to consider deploying all of your company's applications in a single process, isolated by classloaders and security policies. What they never really addressed was the fact that code isn't the only thing you're sharing in this model; you're also sharing memory space, CPU time, and process resources like file descriptors. No amount of Java classloader or security trickery could make this a seamless multiapp environment, and so work like this patent hoped to find a lightweight way for all those child applications to actually live as their own processes.<br /><br />On Android, this manifests in the fact that each application runs independently, and they (like most operating systems) fork off from either the kernel process or some master process.<br /><br />In this case, Oracle's banking on being able to litigate with a patent for a very common application of "fork".<br /><br /><span style="font-weight:bold;"><a href="http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&amp;Sect2=HITOFF&amp;p=1&amp;u=/netahtml/PTO/search-bool.html&amp;r=1&amp;f=G&amp;l=50&amp;co1=AND&amp;d=PTXT&amp;s1=RE38,104.PN.&amp;OS=PN/RE38,104&amp;RS=PN/RE38,104"><span class="Apple-style-span" style="font-size:large;">Method And Apparatus For Resolving Data References In Generated Code (RE38,104)</span></a></span><br /><br />This patent, invented by James Gosling himself, basically describes a mechanism by which symbolic data references in code (e.g. Java field references) can be resolved dynamically at runtime into actual direct memory accesses, eliminating the symbolic lookup overhead. It's part of standard JIT optimization techniques, and there's a lot of references in this patent many great JIT patents and papers of the past.<br /><br />Here there may actually be merit, or as much merit as can be found in a software patent to begin with. The patent itself is tiny, as most of these patents are. The techniques seem obvious to me, but perhaps they're obvious because this patent helped make them standard. I'm not qualified to judge. What I can say is that I can't imagine a VM in existence that doesn't violate the spirit – if not the letter – of this patent as well. All systems with symbolic references will seek to eliminate the symbolic references in favor of direct access. The novelty of this patent may be in doing that translation on the fly...not even at a decidedly coarse-grained per-method level, but by rewriting code while the method is actually executing.<br /><br />I would guess that this is a patent filed during the development of Java's earlier JIT technologies, before systems like Hotspot came along to do a much better large-scale, cross-method job of optimization. It doesn't seem like it would be hard to debunk the novelty of the patent, or at least show prior art that makes it irrelevant.<br /><br /><span style="font-weight:bold;"><span style="font-style:italic;">Update:</span></span> I actually found a reference in the article <a href="http://www.netmite.com/android/mydroid/dalvik/docs/dexopt.html">Dalvik Optimization and Verification with dexopt</a> to the technique described here (about 3/4 down the page, under "Optimization"):<br /><br />"The Dalvik optimizer does the following: ... <span style="font-weight:bold;">For instance field get/put, replace the field index with a byte offset.</span> ..."<br /><br />But Dalvik still does this only once, before running the code (actually, at install time); not *while* running the code as described in the patent.<br /><span style="font-weight:bold;"><a href="http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&amp;Sect2=HITOFF&amp;p=1&amp;u=/netahtml/PTO/search-bool.html&amp;r=1&amp;f=G&amp;l=50&amp;co1=AND&amp;d=PTXT&amp;s1=6,910,205.PN.&amp;OS=PN/6,910,205&amp;RS=PN/6,910,205"><span class="Apple-style-span" style="font-size:large;">Interpreting Functions Utilizing A Hybrid Of Virtual And Native Machine Instructions (6,910,205)</span></a></span><br /><br />This patent, invented by Lars Bak of V8 fame, describes a mechanism for building a "mixed mode" VM that can execute interpreted code and compiled (presumably optimized) code in the same VM process, and flip between the two over time to produce better-optimized compiled code. This describes the basic underpinnings of VMs like Hotspot, which alternate between interpreting virtual machine code and executing real machine code even within the same thread of execution (and sometimes, even branching from virtual code to real code and back within the same method body). Any other VMs that are mixed mode would probably violate this patent, so its impact could reach much farther than Android. (In a sense, even JRuby might violate this patent, though our two mixed modes are both virtual instruction sets.)<br /><br />Now you might think the other mainstream JVMs would violate this patent, but they don't. Neither JRockit nor J9 have interpreters; they both go immediately to native code with various tiers of instrumentation to do the runtime profile data gathering. They iterative regenerate native code with successively more and better optimizations. Lars most recent VM, the V8 Javascript VM at the heart of Chrome, also goes straight to native code.<br /><br />Now here's where it gets weird: Up until Froyo (Android 2.2) Dalvik did a once-only compilation to native code before anything started executing, which means by definition that it was not mixed-mode. And even in Froyo, I believe it still does its initial execution in native code form with instrumentation to allow subsequent compiles to do a better job. Dalvik does not have an interpreter, Dalvik does not interpret Dalvik bytecode.<br /><br />Perhaps someone can explain how this patent even applies to Dalvik or Android?<br /><br /><span style="font-weight: bold; "><span style="font-style: italic; ">Update:</span></span> A couple commenters correct me here: Dalvik actually was 100% interpreted before Froyo, and is now a standard mixed-mode environment post-Froyo. So if this suit had been filed a year ago this patent might not have been applicable, but it probably is now.<br /><br /><span style="font-weight:bold;"><a href="http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&amp;Sect2=HITOFF&amp;p=1&amp;u=/netahtml/PTO/search-bool.html&amp;r=1&amp;f=G&amp;l=50&amp;co1=AND&amp;d=PTXT&amp;s1=6,061,520.PN.&amp;OS=PN/6,061,520&amp;RS=PN/6,061,520"><span class="Apple-style-span" style="font-size:large;">Method And System for Performing Static Initialization (6,061,520)</span></a></span><br /><br />Sigh. This patent appears to revolve completely around a mechanism by which the static initialization of arrays could be "play executed" in a preloader and then rewritten to do static initialization in one shot, or at least more efficiently than running dozens of class initializers that just construct arrays and populate them. Of all the patents, this is probably the narrowest, and the mechanism described are again not very unusual, but there's probably a good chance that the "dex" tool does something along these lines to tidy up static initializers in Android applications.<br /><br />Given the "preloader" aspect of this patent, I'd surmise that it was formulated in part to simplify static initialization of code on embedded devices or in applet environments (because on servers...the boot time of static initialization is probably of little concern). Because of the much more limited nature of embedded environments (especially in 1998, when this patent was filed) it would be very beneficial to turn programmatic data initialization into a simple copy operation or a specialized virtual machine instruction. And this may be why it could apply to Android; it's another sort of embedded Java, with a preloader (either dex or the dexopt tool that jit-compiles your app on the device) and resource limitations that would warrant optimizing static initialization.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">So, Does the Suit Have Merit?</span></span><br /><br />I'll again reiterate that I'm not a lawyer. I'm just a Java developer with a logical mind and a penchant for debunking myths about the Java platform.<br /><br />The collection of patents specified by the suit seems pretty laughable to me. If I were Google, I wouldn't be particularly worried about showing prior art for the patents in question or demonstrating how Android/Dalvik don't actually violate them. Some, like the "mixed mode" patent, don't actually seem to apply at all. It feels very much like a bunch of Sun engineers got together in a room with a bunch of lawyers and started digging for patents that Google might have violated without actually knowing much about Android or Dalvik to begin with.<br /><br />But does the suit have merit? It depends if you consider baseless or over-general patents to have merit. The most substantial patent listed here is the "mixed mode" patent, and unless I'm wrong that one doesn't apply. The others are all variations on prior art, usually specialized for a Java runtime environment (and therefore with some question as to whether they can apply to a non-Java runtime environment that happens to have a translator from Java code). Having read through the suit and scanned the patents, I have to say I'm not particularly worried. But then again, I don't know what sort of magic David Boies and company might be able to pull off.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:x-large;">What Might Happen?</span></span><br /><br />In the unlikely event of a total victory by Oracle, there's probably a lot of possible outcomes. I don't see the "death of Java" among them. There's also the possibility that Google could win a convincing victory. What might happen in each case?<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">The Nuclear Option</span></span><br /><br />The worst case scenario would be that Android is completely destroyed, all Android handsets are confiscated by the Oracle mafia and burned in the city square, and all hope for a Free and Open Java are forever laid to rest. Long live Mono.<br /><br />To understand why this won't happen we need to explore Oracle's possible motives.<br /><br />As I mentioned above, Java ME actually did bring licensing revenue to Sun. There's a lot of handset manufacturers, millions of handsets, and every one put a couple cents (or a couple bucks?) in Sun's pocket. In the heady high times of Java ME, it was the only managed mobile runtime in town, with sound and graphics and standard UI elements. It wasn't always pretty, but it worked well and it was really easy to write for.<br /><br />Now with Android rapidly becoming the preferred mobile and embedded Java, it's become apparent that there's no future for Java ME - or at least no future in the expanding "smart" consumer electronics business. Java ME lives on in Blackberries, some other low-end phones, in most Blu-Ray devices (BD-J is a standard for writing Java apps that run on Blu-Ray systems, utilizing one of the richer class libraries available for Java ME), and in some sub-micro devices like Ajile's AJ-200 Java-based multimedia CPU. If you want Java on a phone or in your TV, Android is taking that world by storm. That means Java ME licensing revenue is rapidly drying up.<br /><br />So why wouldn't Oracle want to take a bite of the rapidly-growing Android pie? Would they turn down a portion of that revenue and instead completely destroy a very popular and successful mobile Java, or would they just strongarm a few bucks out of Google and Android handset manufacturers? Remember we're talking about a profit-driven company here. Java ME is never going to come back to smartphones, that much is certain and I don't think even Oracle could argue it. There's no profit in filing this suit just to kill Android, since it would just mean competing mobile platforms like Windows Phone, RIM, Symbian, or iOS would just canibalize their younger brother. Instead of getting a slice of the fastest-growing segment of Java developers, you'd kill off the entire segment and force those developers to non-Java, non-Oracle-friendly platforms.<br /><br />Oracle may be big and evil, but they're not stupid.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">Google Licensing Deal</span></span><br /><br />A more likely outcome might be that Google would be forced to license the patents or pay royalties on Android revenue. I honestly believe this is the goal of this lawsuit; Oracle wants to get their foot into the door of the smartphone world, and they know they can't innovate enough to make up for the collapse of Java ME. So they're hoping that by sabre-rattling a few patents, Google will be forced (or scared) into sharing the harvest.<br /><br />Given the contents of the suit and the patents, I think this one is pretty unlikely too. Much of Android and Dalvik's designs are specifically crafted to avoid Java entanglements, and I think it's unlikely if this suit goes to trial that Oracle's lawyers would be able to make a convincing argument that the patents were both novel and that they were violated by Google. But let's not put anything past either the lawyers or the US federal court system, eh?<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">Nothing At All</span></span><br /><br />There's a good chance that either Oracle or the court will realize quickly that the case has no merit, and drop all charges. I'm obviously hoping for this one, but it's likely to take the longest of all. First, the court would need to gather all facts in the case, which could take months (especially given the highly technical nature of some of the compaints). Then there's the rebuttals of those facts, sorting out the wheat from the chaff, deciding there's not enough there to proceed, and either Oracle backs out or the court tosses the case. In the latter case, there's the possibility of appeals, and things could start to get very expensive.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:large;">Total Collapse of Software Patents</span></span><br /><br />This is probably the one of the highest-profile cases involving software patents in recent years. The other would be Apple's recent suit against HTC for design elements of the iPhone. Several other bloggers and analysts have called out the possibility that this could lead to the death of software patents in general. I think that's a bit optimistic, but both Google *and* Oracle have come down officially against patents in the past (though perhaps Oracle's had a change of heart since acquiring Sun's portfolio).<br /><br />As much as I'd like to see it happen, software patents probably won't be dead in the next year or two. But this might be a nail in the coffin.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:x-large;">What Does This Mean for Java?</span></span><br /><br />Now we come to the biggest question of all: how does this suit affect the Java world, regardless of outcome?<br /><br />Well it's obviously not great to have two Java heavyweights bickering like schoolchildren, and it would be positively devastating if Android were obliterated because of this. But I think the real damage will be in how the developer community perceives Java, rather than in any lasting impact on the platform itself.<br /><br />Let's return to some of our facts. First off, nothing in this suit would apply to any of the three mainstream JVMs that 99% of the world's Java runs on. Hotspot and JRockit are both owned by Oracle, and J9 is subject to the Java specification's patent grant for compliant implementations. The lesson here is that Android is the first Java-like environment since Microsoft's J++ to attempt to unilaterally subset or superset the platform (with the difference in Android's case being that it doesn't claim to be a Java environment, and it may not actually need the patent grant). Other Java implementations that "follow the Rules" are in the clear, and so 99% of the world's use of Java is in the clear. Sorry, Java haters...this isn't your moment.<br /><br />This certainly does some damage to the notion of open-source Java implementations, but only those that are not (or can not be) compliant with the specification. As the Apache Harmony folks know all too well, it's really hard to build a clean-room implementation of Java and expect to get the "spec compliance patent grant" if you don't actually have the tools necessary to show spec compliance. Tossing the code over to Sun to run compliance testing is a nonstarter; the actual test kit is enormous and requires a huge time investment to set up and run (and Sun/Oracle have better things to do with their time than help out a competing OSS Java implementation). If the test kit had been open-sourced before Sun foundered, there would be no problem; everyone that wanted to make an open-source java would just aim for 100% compliance with the spec and all would be well. As it stands, independently implemented (i.e. non-OpenJDK) open-source Java is a really hard thing to create, especially if you have to clean-room implement all the class libraries yourself. Android has neatly dodged this issue by letting Android just be what it is: a subset of a Java-like platform that doesn't actually run Java bytecode and doesn't use any code from OpenJDK.<br /><br />How will it affect Android if this case drags on? It could certainly hurt Android's adoption by hardware manufacturers, but they're already getting such an oustanding deal on the platform that they might not even care. Android is the first platform that has the potential to unify all hardware profiles, freeing manufacturers from the drudgery of building their own OSes or licensing OSes from someone else. Hell, HTC rose from zero to Hero largely because of their backing of Android and shipping of Android devices. Are they going to back off from that platform now just because Oracle's throwing lawyerbombs at Google? Probably not.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:x-large;">What Does This Mean For You?</span></span><br /><br />If you're a non-Android Java developer...don't lose sleep over this. The details are going to take months to play out, and regardless of the outcome you're probably not going to be affected. Be happy, do great things, and keep making the Java platform a better place.<br /><br />If you're an Android developer...don't lose sleep over this. Even if things go the way of the "Nuclear Option", you've still got a lot of time to build and sell apps and improve yourself as a developer. For a bit of novelty, start considering what a migration path might look like and turn that into a nice Android-agnostic application layer, something that's largely lacking in the current Android APIs. Or explore Android development in languages like JRuby, which are based on off-platform ecosystems that will survive regardless of Android's fate. Whatever you do, don't panic and run for the hills, and don't tell your friends to panic.<br /><br />If you're mad as hell about this...I sympathize. I'm personally going to do whatever I can to keep people informed and keep pushing Android, including but not limited to writing 8000-word essays with my moderately-educated analysis of the "facts". I welcome your help in that fight, and I think it's a damn good time for people that want an open Java and an open mobile platform to show their quality by standing up and letting the world know we're here.<br /><br />"All that is necessary for the triumph of evil is for good men to do nothing."<br /><br />Do something, and we'll get through this together.<br /><br /><span style="font-weight:bold;"><span class="Apple-style-span" style="font-size:x-large;">Footnote: Java Copyrights</span></span><br /><br />I'd love for someone versed in copyright law to provide a brief analysis of how the Java copyrights described (vaguely) in the lawsuit might play out in Android. Java is certainly not ignored as a concept in Android docs, tools, and libraries, but it's unclear to me whether those copyrights amount to something enforceable when it comes to Android or Dalvik.<br /><br /><span style="font-weight:bold;"><span style="font-style:italic;">Update:</span></span> "Crazy" Bob Lee emailed me to clear up a few facts. First off, Android and OpenJDK first came out around roughly the same time, so there was never really time to consider using OpenJDK's GPL'ed class libraries in Android. Bob also claims that Dalvik's design decisions were all technical and not made to circumvent IP, but it seems impossible to me that IP, patent, and licensing issues didn't have *some* influence on those decisions. He goes on to say that Android relies on process separation to sandbox applications, rather than leveraging Java security policies (or similar mechanisms (which Bob insists are badly designed anyway, and I might agree). Finally, he believes that in the worst case scenario, Dalvik would probably only require minor modifications to address the complaints in this suit. The "nuclear option" is, according to Bob, out of the realm of possibility.<br /><br />Thanks for the clarifications, Bob!</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/4344974410425965514/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2010/08/my-thoughts-on-oracle-v-google.html#comment-form' title='74 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/4344974410425965514'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/4344974410425965514'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2010/08/my-thoughts-on-oracle-v-google.html' title='My Thoughts on Oracle v Google'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>74</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-5562710616240372388</id><published>2010-07-19T14:16:00.000-07:00</published><updated>2011-01-25T21:44:30.319-08:00</updated><title type='text'>What JRuby C Extension Support Means to You</title><content type='html'>As part of the Ruby Summer of Code, <a href="http://twitter.com/timfelgentreff">Tim Felgentreff</a> has been building out C extension support for JRuby. He's already made great progress, with simple libraries like Thin and Mongrel working now and larger libraries like RMagick and Yajl starting to function. And we haven't even reached the mid-term evaluation yet. I'd say he gets an "A" so far.<br /><br />I figured it was time I talked a bit about C extensions, what they mean (or don't mean) for JRuby, and how you can help.<br /><br /><span style="font-weight:bold;">The Promise of C Extensions</span><br /><br />One of the "last mile" features keeping people from migrating to JRuby has been their dependence on C extensions that only work on regular Ruby. In some cases, these extensions have been written to improve performance, like the various json libraries. Some of that performance could be less of a concern under Ruby 1.9, but it's hard to claim that any implementation will be able to run Ruby as fast as C for general-purpose libraries any time soon.<br /><br />However, a large number of extensions – perhaps a majority of extensions – exist only to wrap a well-known and well-trusted C library. Nokogiri, for example, wraps the excellent libxml. RMagick wraps ImageMagick. For these cases, there's no alternative on regular Ruby...it's the C library or nothing (or in the case of Nokogiri, your alternatives are only slow and buggy pure-Ruby XML libraries).<br /><br />For the performance case, C extensions on JRuby don't mean a whole lot. In most cases, it would be easier and just as performant to write that code in Java, and many pure-Ruby libraries perform well enough to reduce the need for native code. In addition, there are often libraries that already do what the perf-driven extensions were written for, and it's trivial to just call those libraries directly from Ruby code.<br /><br />But the library case is a bit stickier. Nokogiri does have an FFI version, but it's a maintenance headache for them and a bug report headache for us, due to the lack of a C compiler tying the two halves together. There's a pure-Java Nokogiri in progress, but building both the Ruby bindings and emulating libxml behavior takes a long time to get right. For libraries like RMagick or the native MySQL and SQLite drivers, there are basically no options on the JVM. The Google Summer of Code project RMagick4J, by Sergio Arbeo, was a monumental effort that still has a lot of work left to be done. JDBC libraries work for databases, but they provide a very different interface from the native drivers and don't support things like UNIX domain sockets.<br /><br />There's a very good chance that JRuby C extension support won't perform as well as C extensions on C Ruby, but in many cases that won't matter. Where there's no equivalent library now, having something that's only 5-10x slower to call – but still runs fast and matches API – may be just fine. Think about the coarse-grained operations you feed to a MySQL or SQLite and you get the picture.<br /><br />So ultimately, I think C extensions will be a good thing for JRuby, even if they only serve as a stopgap measure to help people migrate small applications over to native Java equivalents. Why should the end goal be native Java equivalents, you ask?<br /><br /><span style="font-weight:bold;">The Peril of C Extensions</span><br /><br />Now that we're done with the happy, glowing discussion of how great C extension support will be, I can make a confession: I hate C extensions. No feature of C Ruby has done more to hold it back than the desire for backward compatibility with C extensions. Because they have direct pointer access, there's no easy way to build a better garbage collector or easily support multiple runtimes in the same VM, even though various research efforts have tried. I've talked with Koichi Sasada, the creator of Ruby 1.9's "YARV" VM, and there's many things he would have liked to do with YARV that he couldn't because of C extension backward compatibility.<br /><br />For JRuby, supporting C extensions will limit many features that make JRuby compelling in the first place. For example, because C extensions often use a lot of global variables, you can't use them from multiple JRuby runtimes in the same process. Because they expect a Ruby-like threading model, we need to restrict concurrency when calling out from Java to C. And all the great memory tooling I've blogged about recently won't see C extensions or the libraries they call, so it introduces an unknown.<br /><br />All that said, I think it's a good milestone to show that we can support C extensions, and it may make for a "better JNI" for people who really just want to write C or who simply need to wrap a native library.<br /><br /><span style="font-weight:bold;">How You Can Help</span><br /><br />There's a few things I think users like you can help with.<br /><br />First off, we'd love to know what extensions you are using today, so we can explore what it would take to run them under JRuby (and so we can start exploring pure-Java alternatives, too.) Post your list in the comments, and we'll see what we can come up with.<br /><br />Second, anyone that knows C and the Ruby C API (like folks who work on extensions) could help us fill out bits and pieces that are missing. Set up the JRuby cext branch (I'll show you how in a moment), and try to get your extensions to build and load. Tim has already done the heavy lifting of making "gem install xyz" attempt to build the extension and "require 'xyz'" try to load the resulting native library, so you can follow the usual processes (including extconf.rb/mkmf.rb for non-gem building and testing.) If it doesn't build ok, help us figure out what's missing or incorrect. If it builds but doesn't run, help us figure out what it's doing incorrectly.<br /><br /><span style="font-weight:bold;">Building JRuby with C Extension Support</span><br /><br />Like building JRuby proper, building the cext work is probably the easiest thing you'll do all day (assuming the C compiler/build/toolchain doesn't bite you.<br /><ol><li>Check out (or fork and check out) the JRuby repository from <a href="http://github.com/jruby/jruby">http://github.com/jruby/jruby</a>:<br /><pre>git clone git://github.com/jruby/jruby.git</pre></li><br /><li>Switch to the "cext" branch:<br /><pre>git checkout -b cext origin/cext</pre></li><br /><li>Do a clean build of JRuby plus the cext subsystem:<br /><pre>ant clean build-jruby-cext-native</pre></li></ol><br />At this point you should have a JRuby build (run with bin/jruby) that can gem install and load native extensions.</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/5562710616240372388/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2010/07/what-jruby-c-extension-support-means-to.html#comment-form' title='14 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5562710616240372388'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/5562710616240372388'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2010/07/what-jruby-c-extension-support-means-to.html' title='What JRuby C Extension Support Means to You'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>14</thr:total></entry><entry><id>tag:blogger.com,1999:blog-4704664917418794835.post-6351925351093074122</id><published>2010-07-17T14:58:00.000-07:00</published><updated>2011-01-25T21:44:30.373-08:00</updated><title type='text'>Browsing Memory with Ruby and Java Debug Interface</title><content type='html'>This is the third post in a series. The first two were on <a href="http://blog.headius.com/2010/07/browsing-memory-jruby-way.html">Browsing Memory the JRuby Way</a> and <a href="http://blog.headius.com/2010/07/finding-leaks-in-ruby-apps-with-eclipse.html">Finding Leaks in Ruby Apps with Eclipse Memory Analyzer</a><br /><br />Hello again, friends! I'm back with more exciting memory analysis tips and tricks! Ready? Here we go!<br /><br />After my previous two posts, several folks asked if it's possible to do all this stuff from Ruby, rather than using Java or C-based apps shipped with the JVM. The answer is yes! Because of the maturity of the Java platform, there are standard Java APIs you can use to access all the same information the previous tools consumed. And since we're talking about JRuby, that means you have Ruby APIs you can use to access that information.<br /><br />That's what I'm going to show you today.<br /><br /><span style="font-weight:bold;">Introducing JDI</span><br /><br />The APIs we'll be using are part of the <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/jdk/api/jpda/jdi/index.html">Java Debug Interface (JDI)</a>, a set of Java APIs for remotely inspecting a running application. It's part of the <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/technotes/guides/jpda/">Java Platform Debugger Architecture</a>, which also includes a <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/technotes/guides/jvmti/index.html">C/++ API</a>, a <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/technotes/guides/jpda/jdwp-spec.html">wire protocol</a>, and a raw <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/technotes/guides/jpda/jdwpTransport.html">wire protocol API</a>. Exploring those is left as an exercise for the reader...but they're also pretty cool.<br /><br />We'll use the Rails app from before, inspecting it immediately after boot. JDI provides a number of ways to connect up to a running VM, using <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/jdk/api/jpda/jdi/com/sun/jdi/VirtualMachineManager.html">VirtualMachineManager</a>; you can either have the debugger make the connection or the target VM make the connection, and optionally have the target VM launch the debugger or the debugger launch the target VM. For our example, we'll have the debugger attach to a target VM listening for connections.<br /><br /><span style="font-weight:bold;">Preparing the Target VM</span><br /><br />The first step is to start up the application with the appropriate debugger endpoint installed. This new flag is a bit of a mouthful (and we should make a standard flag for JRuby users), but we're simply setting up a socket-based listener on port 12345, running as a server, and we don't want to suspend the JVM when the debugger connects.<br /><pre>jruby -J-agentlib:jdwp=transport=dt_socket,server=y,address=12345,suspend=n -J-Djruby.reify.classes=true script/server -e production</pre><br />The -J-Djruby.reify.classes bit I talked about in my first post. It makes Ruby classes show up as Java classes for purposes of heap inspection.<br /><br />The rest is just running the server in production mode.<br /><br />As you can see, remote debugging is already baked into the JVM, which means we didn't have to write it or debug it. And that's pretty awesome.<br /><br />Let's connect to our Rails process and see what we can do.<br /><br /><span style="font-weight:bold;">Connecting to the target VM</span><br /><br />In order to connect to the target VM, you need to do the Java factory dance. We start with the <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/jdk/api/jpda/jdi/com/sun/jdi/Bootstrap.html">com.sun.jdi.Bootstrap class</a>, get a <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/jdk/api/jpda/jdi/com/sun/jdi/VirtualMachineManager.html">com.sun.jdi.VirtualMachineManager</a>, and then connect to a target VM to get a <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/jdk/api/jpda/jdi/com/sun/jdi/VirtualMachine.html">com.sun.jdi.VirtualMachine</a> object.<br /><pre>vmm = Bootstrap.virtual_machine_manager<br />sock_conn = vmm.attaching_connectors[0] # not guaranteed to be Socket<br />args = sock_conn.default_arguments<br />args['hostname].value = "localhost"<br />args['port'].value = "12345"<br />vm = sock_conn.attach(args)</pre><br />Notice that I didn't dig out the socket connector explicitly here, because on my system, the first connector always appears to be the socket connector. Here's the full list for me on OS X:<br /><pre>➔ jruby -rjava -e "puts com.sun.jdi.Bootstrap.virtual_machine_manager.attaching_connectors<br />> "<br />[com.sun.jdi.SocketAttach (defaults: timeout=, hostname=charles-nutters-macbook-pro.local, port=),<br />com.sun.jdi.ProcessAttach (defaults: pid=, timeout=)]</pre><br />The ProcessAttach connector there isn't as magical as it looks; all it does is query the target process to find out what transport it's using (dt_socket in our case) and then calls the right connector (e.g. SocketAttach in the case of dt_socket or SharedMemoryAttach if you use dt_shmem on Windows). In our case, we know it's listening on a socket, so we're using the SocketAttach connector directly.<br /><br />The rest is pretty simple: we get the default arguments from the connector, twiddle them to have the right hostname and port number, and attach to the VM. Now we have a VirtualMachine object we can query and twiddle; we're inside the matrix.<br /><br /><span style="font-weight:bold;">With Great Power...</span><br /><br />So, what can we do with this VirtualMachine object? We can:<br /><ul><li>walk all classes and objects on the heap</li><li>install breakpoints and step-debug any running code</li><li>inspect and modify the current state of any running thread, even manipulating in-flight arguments and variables</li><li>replace already-loaded classes with new definitions (such as to install custom instrumentation)</li></ul>Here's the output from JRuby's ri command when we ask about VirtualMachine:<br /><pre>➔ ri --java com.sun.jdi.VirtualMachine<br />-------------------------------------- Class: com.sun.jdi.VirtualMachine<br /> (no description...)<br />------------------------------------------------------------------------<br /><br /><br />Instance methods:<br />-----------------<br /> allClasses, allThreads, canAddMethod, canBeModified,<br /> canForceEarlyReturn, canGetBytecodes, canGetClassFileVersion,<br /> canGetConstantPool, canGetCurrentContendedMonitor,<br /> canGetInstanceInfo, canGetMethodReturnValues,<br /> canGetMonitorFrameInfo, canGetMonitorInfo, canGetOwnedMonitorInfo,<br /> canGetSourceDebugExtension, canGetSyntheticAttribute, canPopFrames,<br /> canRedefineClasses, canRequestMonitorEvents,<br /> canRequestVMDeathEvent, canUnrestrictedlyRedefineClasses,<br /> canUseInstanceFilters, canUseSourceNameFilters,<br /> canWatchFieldAccess, canWatchFieldModification, classesByName,<br /> description, dispose, eventQueue, eventRequestManager, exit,<br /> getDefaultStratum, instanceCounts, mirrorOf, mirrorOfVoid, name,<br /> process, redefineClasses, resume, setDebugTraceMode,<br /> setDefaultStratum, suspend, toString, topLevelThreadGroups,<br /> version, virtualMachine</pre><br />We can basically make the target VM dance any way we want, even going so far as to write our own debugger entirely in Ruby code. But that's a topic for another day. Right now, we're going to do some memory inspection.<br /><br /><b>Creating a Histogram of the Heap</b><br />The simplest heap inspection we might do is to produce a histogram of all objects on the heap. And as you might expect, this is one of the easiest things to do, because it's the first thing everyone looks for when debugging a memory issue.<br /><pre>classes = VM.all_classes<br />counts = VM.instance_counts(classes)<br />classes.zip(counts)</pre><br />VirtualMachine.all_classes gives you a list (a java.util.List, but we make those behave mostly like a Ruby Array) of every class the JVM has loaded, including Ruby classes, JRuby core and runtime classes, and other Java classes that JRuby and the JVM use. VirtualMachine.instance_counts takes that list of classes and returns another list of instance counts. Zip the two together, and we have an array of classes and instance counts. So easy!<br /><br />Let's take these two pieces and put them together in an easy-to-use class<br /><pre>require 'java'<br /><br />module JRuby<br /> class Debugger<br /> VMM = com.sun.jdi.Bootstrap.virtual_machine_manager<br /> <br /> attr_accessor :vm<br /> <br /> def initialize(options = {})<br /> connectors = VMM.attaching_connectors<br /> if options[:port]<br /> connector = connectors.find {|ac| ac.name =~ /Socket/}<br /> elsif options[:pid]<br /> connector = connectors.find {|ac| ac.name =~ /Process/}<br /> end<br /><br /> args = connector.default_arguments<br /> for k, v in options<br /> args[k.to_s].value = v.to_s<br /> end<br /> <br /> @vm = connector.attach(args)<br /> end<br /><br /> # Generate a histogram of all classes in the system<br /> def histogram<br /> classes = @vm.all_classes<br /> counts = @vm.instance_counts(classes)<br /> classes.zip(counts)<br /> end<br /> end<br />end</pre><br />I've taken the liberty of expanding the connection process to handle pids and other arguments passed in. So to get a histogram from a VM listening on localhost port 12345, we can simply do:<br /><pre>JRuby::Debugger.new(:hostname => 'localhost', :port => 12345).histogram</pre><br />Now of course this list is going to have a lot of JRuby and Java objects that we might not be interested in, so we'll want to filter it to just the Ruby classes. On JRuby master, all the generated Ruby classes start with a package name "ruby". Unfortunately, jitted Ruby methods start with a package of "ruby.jit" right now, so we'll want to filter those out too (unless you're interested in them, of course...JRuby is an open book!)<br /><pre>require 'jruby_debugger'<br /><br /># connect to the VM<br />debugr = JRuby::Debugger.new(:hostname => 'localhost', :port => 12345)<br />histo = debugr.histogram<br /># sort by count<br />histo.sort! {|a,b| b[1] <=> a[1]}<br /># filter to only user-created Ruby classes with >0 instances<br />histo.each do |cls,num|<br /> next if num == 0 || cls.name[0..4] != 'ruby.' || cls.name[5..7] == 'jit'<br /> puts "#{num} instances of #{cls.name[5..-1].gsub('.', '::')}"<br />end</pre><br />If we run this short script against our Rails application, we see similar results to the previous posts (but it's cooler, because we're doing it all from Ruby!)<br /><pre>➔ jruby ruby_histogram.rb | head -10<br />11685 instances of TZInfo::TimezoneTransitionInfo<br />1071 instances of Gem::Version<br />1012 instances of Gem::Requirement<br />592 instances of TZInfo::TimezoneOffsetInfo<br />432 instances of Gem::Dependency<br />289 instances of Gem::Specification<br />142 instances of ActiveSupport::TimeZone<br />118 instances of TZInfo::DataTimezoneInfo<br />118 instances of TZInfo::DataTimezone<br />45 instances of Gem::Platform</pre><br />Just so we're all on the same page, it's important to know what we're actually dealing with here. VirtualMachine.all_classes returns a list of <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/jdk/api/jpda/jdi/com/sun/jdi/ReferenceType.html">com.sun.jdi.ReferenceType</a> objects. Let's ri that.<br /><pre>➔ ri --java com.sun.jdi.ReferenceType<br />--------------------------------------- Class: com.sun.jdi.ReferenceType<br /> (no description...)<br />------------------------------------------------------------------------<br /><br /><br />Instance methods:<br />-----------------<br /> allFields, allLineLocations, allMethods, availableStrata,<br /> classLoader, classObject, compareTo, constantPool,<br /> constantPoolCount, defaultStratum, equals, failedToInitialize,<br /> fieldByName, fields, genericSignature, getValue, getValues,<br /> hashCode, instances, isAbstract, isFinal, isInitialized,<br /> isPackagePrivate, isPrepared, isPrivate, isProtected, isPublic,<br /> isStatic, isVerified, locationsOfLine, majorVersion, methods,<br /> methodsByName, minorVersion, modifiers, name, nestedTypes,<br /> signature, sourceDebugExtension, sourceName, sourceNames,<br /> sourcePaths, toString, virtualMachine, visibleFields,<br /> visibleMethods</pre><br />You can see there's quite a bit more you can do with a ReferenceType. Let's try something.<br /><br /><span style="font-weight:bold;">Digging Deeper Into TimezoneTransitionInfo</span><br /><br />Let's actually take some time to explore our old friend TimezoneTransitionInfo (hereafter referred to as TTI). Instead of walking all classes in the system, we'll want to just grab TTI directly. For that we use VirtualMachine.classes_by_name, which returns a list of classes on the target VM of that name. There should be only one, since we only have a single JRuby instance in our server, so we'll grab that class and request exactly one instance of it...any old instance.<br /><pre>tti_class = debugr.vm.classes_by_name('ruby.TZInfo.TimezoneTransitionInfo')[0]<br />tti_obj = tti_class.instances(1)[0]<br />puts tti_obj</pre><br />Running this we can see we've got the reference we're looking for.<br /><pre>➔ jruby tti_digger.rb<br />instance of ruby.TZInfo.TimezoneTransitionInfo(id=2)</pre><br />ReferenceType.instances returns a list (no larger than the specified size, or all instances if you specify 0) of <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/jdk/api/jpda/jdi/com/sun/jdi/ObjectReference.html">com.sun.jdi.ObjectReference</a> objects.<br /><pre>➔ ri --java com.sun.jdi.ObjectReference<br />------------------------------------- Class: com.sun.jdi.ObjectReference<br /> (no description...)<br />------------------------------------------------------------------------<br /><br /><br />Instance methods:<br />-----------------<br /> disableCollection, enableCollection, entryCount, equals, getValue,<br /> getValues, hashCode, invokeMethod, isCollected, owningThread,<br /> referenceType, referringObjects, setValue, toString, type,<br /> uniqueID, virtualMachine, waitingThreads</pre><br />Among the weirder things like disabling garbage collection for this object or listing all threads waiting on this object's monitor (a la 'synchronize' in Java), we can access the object's fields through getValue and setValue.<br /><br />Let's examine the instance variables TTI contains. You may recall from previous posts that all Ruby objects in JRuby store their instance variables in an array, to avoid the large memory and cpu cost of storing them in a map. We can grab a reference to that array and display its contents.<br /><pre>var_table_field = tti_class.field_by_name('varTable')<br />tti_vars = tti_obj.get_value(var_table_field)<br />puts "varTable: #{tti_vars}"<br />puts tti_vars.values.map(&:to_s)</pre><br />And the new output:<br /><pre>➔ jruby tti_digger.rb<br />varTable: instance of java.lang.Object[7] (id=13)<br />instance of ruby.TZInfo.TimezoneOffsetInfo(id=15)<br />instance of ruby.TZInfo.TimezoneOffsetInfo(id=16)<br />instance of org.jruby.RubyFixnum(id=17)<br />instance of org.jruby.RubyFixnum(id=18)<br />instance of org.jruby.RubyNil(id=19)<br />instance of org.jruby.RubyNil(id=19)<br />instance of org.jruby.RubyNil(id=19)</pre><br />Since the varTable field is a simple Object[] in Java, the reference we get to it is of type <a href="http://download.oracle.com/docs/cd/E17409_01/javase/6/docs/jdk/api/jpda/jdi/com/sun/jdi/ArrayReference.html">com.sun.jdi.ArrayReference</a>.<br /><pre>➔ ri --java com.sun.jdi.ArrayReference<br />-------------------------------------- Class: com.sun.jdi.ArrayReference<br /> (no description...)<br />------------------------------------------------------------------------<br /><br /><br />Instance methods:<br />-----------------<br /> disableCollection, enableCollection, entryCount, equals, getValue,<br /> getValues, hashCode, invokeMethod, isCollected, length,<br /> owningThread, referenceType, referringObjects, setValue, setValues,<br /> toString, type, uniqueID, virtualMachine, waitingThreads</pre><br />Of course each of these references can be further explored, but already we can see that this TTI instance has seven instance variables: two TimezoneOffsetInfo objects, two Fixnums, and three nils. But we don't have instance variable names!<br /><br />Instance variable names are only stored on the object's class. There, a table of names to offsets is kept up-to-date as new instance variable names are discovered. We can access this from the TTI class reference and combine it with the variable table to get the output we want to see.<br /><pre># get the metaclass object and class reference<br />metaclass_field = tti_class.field_by_name('metaClass')<br />tti_class_obj = tti_obj.get_value(metaclass_field)<br />tti_class_class = tti_class_obj.reference_type<br /><br /># get the variable names from the metaclass object<br />var_names_field = tti_class_class.field_by_name('variableNames')<br />var_names = tti_class_obj.get_value(var_names_field)<br /><br /># splice the names and values together<br />table = var_names.values.zip(tti_vars.values)<br />puts table</pre><br />This looks a bit complicated, but there's actually a lot of boilerplate here we could put into a utility class. For example, the metaClass and variableNames fields are standard on all (J)Ruby objects and classes, respectively. But considering that we're actually walking a remote VM's *live* heap...this is pretty simple code.<br /><br />Here's what our script outputs now:<br /><pre>➔ jruby tti_digger.rb<br />"@offset"<br />instance of ruby.TZInfo.TimezoneOffsetInfo(id=25)<br />"@previous_offset"<br />instance of ruby.TZInfo.TimezoneOffsetInfo(id=26)<br />"@numerator_or_time"<br />instance of org.jruby.RubyFixnum(id=27)<br />"@denominator"<br />instance of org.jruby.RubyFixnum(id=28)<br />"@at"<br />instance of org.jruby.RubyNil(id=29)<br />"@local_end"<br />instance of org.jruby.RubyNil(id=29)<br />"@local_start"<br />instance of org.jruby.RubyNil(id=29)</pre><br />We could go even deeper, but I think you get the idea.<br /><br /><span style="font-weight:bold;">Your Turn</span><br /><br />Here's a <a href="http://gist.github.com/481102">gist of the three scripts we've created</a>, so you can refer to and build off of them. And of course the javadocs and ri docs will help you as well, plus everything we've done here you can do in a jirb session.<br /><br />There's a lot to the JDI API, but once you've got the VirtualMachine object in hand it's pretty easy to follow. As you'd expect from any debugger API, you need to know a bit about how things work on the inside, but through the magic of JRuby it's actually possible to write most of those fancy memory and debugging tools entirely in Ruby. Perhaps this article has peaked your interest in exploring JRuby internals using JDI and you might start to write debugging tools. Perhaps we can ship a few utilities to make some of the boilerplate go away. In any case, I hope this series of articles shows that JRuby users have an amazing library of tools available to them, and you don't even have to leave your comfort zone if you don't want to.<br /><br />Note: The variableNames field is a recent addition to JRuby master, so if you'd like to play with that you'll probably want to build JRuby yourself or wait for a nightly build that picks it up. But you can certainly do a lot of exploring even without that patch.</content><link rel='replies' type='application/atom+xml' href='http://blog.headius.com/feeds/6351925351093074122/comments/default' title='Post Comments'/><link rel='replies' type='text/html' href='http://blog.headius.com/2010/07/browsing-memory-with-ruby-and-java.html#comment-form' title='1 Comments'/><link rel='edit' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/6351925351093074122'/><link rel='self' type='application/atom+xml' href='http://www.blogger.com/feeds/4704664917418794835/posts/default/6351925351093074122'/><link rel='alternate' type='text/html' href='http://blog.headius.com/2010/07/browsing-memory-with-ruby-and-java.html' title='Browsing Memory with Ruby and Java Debug Interface'/><author><name>Charles Nutter</name><uri>https://plus.google.com/101599370339210456684</uri><email>noreply@blogger.com</email><gd:image rel='http://schemas.google.com/g/2005#thumbnail' width='32' height='32' src='//lh5.googleusercontent.com/-VT5b8KsgHgQ/AAAAAAAAAAI/AAAAAAAAFB0/9d5SU9AcHNQ/s512-c/photo.jpg'/></author><thr:total>1</thr:total>
|
|
104
|
+
</entry>
|
|
105
|
+
</feed>
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
feed.format: atom
|
|
110
|
+
feed.title: Headius
|
|
111
|
+
feed.summary: Helping the JVM Into the 21st Century
|
|
112
|
+
feed.url: http://blog.headius.com/
|
|
113
|
+
feed.generator: Blogger
|
|
114
|
+
feed.generator_uri: http://www.blogger.com
|
|
115
|
+
feed.generator_version: 7.00
|
|
116
|
+
|
|
117
|
+
feed.items[0].title: JRubyConf.eu 2014!
|
|
118
|
+
feed.items[0].url: http://blog.headius.com/2014/05/jrubyconfeu-2014.html
|
|
119
|
+
feed.items[0].guid: tag:blogger.com,1999:blog-4704664917418794835.post-3430080308857860963
|
|
120
|
+
|
|
121
|
+
feed.items[1].title: The Pain of Broken Subprocess Management on JDK
|
|
122
|
+
feed.items[1].guid: tag:blogger.com,1999:blog-4704664917418794835.post-462657466694269626
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="utf-8"?>
|
|
2
|
+
<rss version="2.0" xml:base="http://lambda-the-ultimate.org">
|
|
3
|
+
<channel>
|
|
4
|
+
<title>Lambda the Ultimate - Programming Languages Weblog</title>
|
|
5
|
+
<link>http://lambda-the-ultimate.org</link>
|
|
6
|
+
<description>Programming languages news, articles and discussion</description>
|
|
7
|
+
<language>en</language>
|
|
8
|
+
<item>
|
|
9
|
+
<title> John C Reynolds Doctoral Dissertation Award nominations for 2014</title>
|
|
10
|
+
<link>http://lambda-the-ultimate.org/node/5088</link>
|
|
11
|
+
<description><blockquote ><p >Presented annually to the author of the outstanding doctoral dissertation in the area of Programming Languages. The award includes a prize of $1,000. The winner can choose to receive the award at ICFP, OOPSLA, POPL, or PLDI.</blockquote>
|
|
12
|
+
<p >
|
|
13
|
+
I guess it is fairly obvious why professors should <a href="http://www.sigplan.org/Awards/Dissertation/">propose their students</a> (the deadline is January 4th 2015). Newly minted PhD should, for similar reasons, make sure their professors are reminded of these reasons. I can tell you that the competition is going to be tough this year; but hey, you didn't go into programming language theory thinking it is going to be easy, did you? </p></description>
|
|
14
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/6">General</category>
|
|
15
|
+
<pubDate>Wed, 26 Nov 2014 22:05:25 +0000</pubDate>
|
|
16
|
+
</item>
|
|
17
|
+
<item>
|
|
18
|
+
<title>Zélus : A Synchronous Language with ODEs</title>
|
|
19
|
+
<link>http://lambda-the-ultimate.org/node/5087</link>
|
|
20
|
+
<description><p >
|
|
21
|
+
<a href="http://www.di.ens.fr/~pouzet/bib/hscc13.pdf">
|
|
22
|
+
Zélus : A Synchronous Language with ODEs</a><br >
|
|
23
|
+
Timothy Bourke, Marc Pouzet<br >
|
|
24
|
+
2013</p>
|
|
25
|
+
|
|
26
|
+
<blockquote >
|
|
27
|
+
<p >
|
|
28
|
+
<a href="http://zelus.di.ens.fr/">Zélus</a> is a new programming
|
|
29
|
+
language for modeling systems that mix discrete logical time and
|
|
30
|
+
continuous time behaviors. From a user's perspective, its main
|
|
31
|
+
originality is to extend an existing <a href="http://en.wikipedia.org/wiki/Lustre_%28programming_language%29">Lustre</a>-like
|
|
32
|
+
synchronous language with Ordinary Differential Equations (ODEs). The
|
|
33
|
+
extension is conservative: any synchronous program expressed as
|
|
34
|
+
data-flow equations and hierarchical automata can be composed
|
|
35
|
+
arbitrarily with ODEs in the same source code.</p>
|
|
36
|
+
|
|
37
|
+
<p >A dedicated type system and causality analysis ensure that all
|
|
38
|
+
discrete changes are aligned with zero-crossing events so that no side
|
|
39
|
+
effects or discontinuities occur during integration. Programs are
|
|
40
|
+
statically scheduled and translated into sequential code that, by
|
|
41
|
+
construction, runs in bounded time and space. Compilation is effected
|
|
42
|
+
by source-to-source translation into a small synchronous subset which
|
|
43
|
+
is processed by a standard synchronous compiler architecture. The
|
|
44
|
+
resultant code is paired with an off-the-shelf numeric solver.</p>
|
|
45
|
+
|
|
46
|
+
<p >We show that it is possible to build a modeler for explicit hybrid
|
|
47
|
+
systems à la <a href="http://en.wikipedia.org/wiki/Simulink">Simulink</a>/<a href="http://en.wikipedia.org/wiki/Stateflow">Stateflow</a> on top of
|
|
48
|
+
an existing synchronous language, using it both as a semantic basis
|
|
49
|
+
and as a target for code generation.</p>
|
|
50
|
+
</blockquote>
|
|
51
|
+
|
|
52
|
+
<p >Synchronous programming languages (à la <a href="http://www.di.ens.fr/~pouzet/lucid-synchrone/">Lucid
|
|
53
|
+
Synchrone</a>) are language designs for reactive systems with discrete
|
|
54
|
+
time. Zélus extends them gracefully to hybrid discrete/continuous
|
|
55
|
+
systems, to interact with the physical world, or simulate it -- while
|
|
56
|
+
preserving their strong semantic qualities.</p>
|
|
57
|
+
|
|
58
|
+
<p >The paper is short (6 pages) and centered around examples rather than
|
|
59
|
+
the theory -- I enjoyed it. Not being familiar with the domain, I was
|
|
60
|
+
unsure what the "zero-crossings" mentioned in the introductions are,
|
|
61
|
+
but there is a good explanation further down in the paper:</p>
|
|
62
|
+
|
|
63
|
+
<blockquote ><p >The standard way to detect events in a numeric solver is
|
|
64
|
+
via zero-crossings where a solver monitors expressions for changes in
|
|
65
|
+
sign and then, if they are detected, searches for a more precise
|
|
66
|
+
instant of crossing.</p></blockquote>
|
|
67
|
+
|
|
68
|
+
<p >The <a href="http://zelus.di.ens.fr/">Zélus website</a> has a <a href="http://zelus.di.ens.fr/papers.html">'publications' page</a> with
|
|
69
|
+
more advanced material, and an <a href="http://zelus.di.ens.fr/examples.html">'examples' page</a> with
|
|
70
|
+
case studies.</p></description>
|
|
71
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/6">General</category>
|
|
72
|
+
<pubDate>Sat, 22 Nov 2014 18:31:27 +0000</pubDate>
|
|
73
|
+
</item>
|
|
74
|
+
<item>
|
|
75
|
+
<title>Facebook releases "Flow", a statically typed JavaScript variant</title>
|
|
76
|
+
<link>http://lambda-the-ultimate.org/node/5086</link>
|
|
77
|
+
<description><blockquote ><p >
|
|
78
|
+
The goal of Flow is to find errors in JavaScript code with little programmer effort. Flow relies heavily on type inference to find type errors even when the program has not been annotated - it precisely tracks the types of variables as they flow through the program.</p>
|
|
79
|
+
<p >At the same time, Flow is a gradual type system. Any parts of your program that are dynamic in nature can easily bypass the type checker, so you can mix statically typed code with dynamic code.</p>
|
|
80
|
+
<p >Flow also supports a highly expressive type language. Flow types can express much more fine-grained distinctions than traditional type systems. For example, Flow helps you catch errors involving null, unlike most type systems.
|
|
81
|
+
</p></blockquote>
|
|
82
|
+
<p ><a href="http://flowtype.org/docs/about-flow.html#_">Read more here.</a><br >
|
|
83
|
+
<a href="https://code.prod.facebook.com/posts/1505962329687926/flow-a-new-static-type-checker-for-javascript/">Here's the announcement from Facebook.</a></p></description>
|
|
84
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/6">General</category>
|
|
85
|
+
<pubDate>Tue, 18 Nov 2014 19:19:15 +0000</pubDate>
|
|
86
|
+
</item>
|
|
87
|
+
<item>
|
|
88
|
+
<title>Why do we need modules at all?</title>
|
|
89
|
+
<link>http://lambda-the-ultimate.org/node/5079</link>
|
|
90
|
+
<description><p ><a href="http://erlang.org/pipermail/erlang-questions/2011-May/058768.html">Post</a> by Joe Armstrong of Erlang fame. Leader:</p>
|
|
91
|
+
<blockquote ><p >
|
|
92
|
+
Why do we need modules at all? This is a brain-dump-stream-of-consciousness-thing. I've been thinking about this for a while. I'm proposing a slightly different way of programming here. The basic idea is:</p>
|
|
93
|
+
<ul >
|
|
94
|
+
<li > do away with modules
|
|
95
|
+
<li > all functions have unique distinct names
|
|
96
|
+
<li > all functions have (lots of) meta data
|
|
97
|
+
<li > all functions go into a global (searchable) Key-value database
|
|
98
|
+
<li > we need letrec
|
|
99
|
+
<li > contribution to open source can be as simple as contributing a single function
|
|
100
|
+
<li > there are no "open source projects" - only "the open source Key-Value database of all functions"
|
|
101
|
+
<li > Content is peer reviewed
|
|
102
|
+
</ul>
|
|
103
|
+
<p >Why does Erlang have modules? There's a good an bad side to modules. Good: Provides a unit of compilation, a unit of code distribution. unit of code replacement. Bad: It's very difficult to decide which module to put an individual function in. Break encapsulation (see later).
|
|
104
|
+
</p></blockquote></description>
|
|
105
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/1">LtU Forum</category>
|
|
106
|
+
<pubDate>Wed, 05 Nov 2014 23:56:11 +0000</pubDate>
|
|
107
|
+
</item>
|
|
108
|
+
<item>
|
|
109
|
+
<title>Conservation laws for free!</title>
|
|
110
|
+
<link>http://lambda-the-ultimate.org/node/5078</link>
|
|
111
|
+
<description><p >In this year's <A href="http://popl.mpi-sws.org/2014/">POPL</A>, <A href="bentnib.org">Bob Atkey</A> made a splash by showing how to get <A href="http://bentnib.org/conservation-laws.pdf">from parametricity to conservation laws, via Noether's theorem</A>:</p>
|
|
112
|
+
<blockquote ><p >
|
|
113
|
+
Invariance is of paramount importance in programming languages and in physics. In programming languages, John Reynolds’ theory of relational parametricity demonstrates that parametric polymorphic programs are invariant under change of data representation, a property that yields “free” theorems about programs just from their types. In physics, Emmy Noether showed that if the action of a physical system is invariant under change of coordinates, then the physical system has a conserved quantity: a quantity that remains constant for all time. Knowledge of conserved quantities can reveal deep properties of physical systems. For example, the conservation of energy, which by Noether’s theorem is a consequence of a system’s invariance under time-shifting.</p>
|
|
114
|
+
<p > In this paper, we link Reynolds’ relational parametricity with Noether’s theorem for deriving conserved quantities. We propose an extension of System Fω with new kinds, types and term constants for writing programs that describe classical mechanical systems in terms of their Lagrangians. We show, by constructing a relationally parametric model of our extension of Fω, that relational parametricity is enough to satisfy the hypotheses of Noether’s theorem, and so to derive conserved quantities for free, directly from the polymorphic types of Lagrangians expressed in our system.
|
|
115
|
+
</p></blockquote></description>
|
|
116
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/22">Category Theory</category>
|
|
117
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/5">Fun</category>
|
|
118
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/11">Functional</category>
|
|
119
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/20">Lambda Calculus</category>
|
|
120
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/34">Scientific Programming</category>
|
|
121
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/29">Semantics</category>
|
|
122
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/19">Theory</category>
|
|
123
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/21">Type Theory</category>
|
|
124
|
+
<pubDate>Tue, 28 Oct 2014 07:52:46 +0000</pubDate>
|
|
125
|
+
</item>
|
|
126
|
+
<item>
|
|
127
|
+
<title>Seemingly impossible programs</title>
|
|
128
|
+
<link>http://lambda-the-ultimate.org/node/5074</link>
|
|
129
|
+
<description><p >In case this one went under the radar, at <A href="http://www.cse.psu.edu/popl/12/">POPL'12</A>, <A href="http://www.cs.bham.ac.uk/~mhe/">Martín Escardó</A> gave a tutorial on <A href="http://www.cs.bham.ac.uk/~mhe/.talks/popl2012/escardo-popl2012.pdf">seemingly impossible functional programs</A>:</p>
|
|
130
|
+
<blockquote ><p >
|
|
131
|
+
Programming language semantics is typically applied to<br >
|
|
132
|
+
prove compiler correctness and allow (manual or automatic) program<br >
|
|
133
|
+
verification. Certain kinds of semantics can also be applied to<br >
|
|
134
|
+
discover programs that one wouldn't have otherwise thought of. This is<br >
|
|
135
|
+
the case, in particular, for semantics that incorporate topological<br >
|
|
136
|
+
ingredients (limits, continuity, openness, compactness). For example,<br >
|
|
137
|
+
it turns out that some function types (X -&gt; Y) with X infinite (but<br >
|
|
138
|
+
compact) do have decidable equality, contradicting perhaps popular<br >
|
|
139
|
+
belief, but certainly not (higher-type) computability theory. More<br >
|
|
140
|
+
generally, one can often check infinitely many cases in finite time.</p>
|
|
141
|
+
<p >I will show you such programs, run them fast in surprising instances,<br >
|
|
142
|
+
and introduce the theory behind their derivation and working. In<br >
|
|
143
|
+
particular, I will study a single (very high type) program that (i)<br >
|
|
144
|
+
optimally plays sequential games of unbounded length, (ii) implements<br >
|
|
145
|
+
the Tychonoff Theorem from topology (and builds finite-time search<br >
|
|
146
|
+
functions for infinite sets), (iii) realizes the double-negation shift<br >
|
|
147
|
+
from proof theory (and allows us to extract programs from classical<br >
|
|
148
|
+
proofs that use the axiom of countable choice). There will be several<br >
|
|
149
|
+
examples in the languages Haskell and Agda.
|
|
150
|
+
</p></blockquote>
|
|
151
|
+
<p >A <A href="http://math.andrej.com/2007/09/28/seemingly-impossible-functional-programs/">shorter version</A> (coded in Haskell) appears in Andrej Bauer's blog.</p></description>
|
|
152
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/22">Category Theory</category>
|
|
153
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/5">Fun</category>
|
|
154
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/11">Functional</category>
|
|
155
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/10">Paradigms</category>
|
|
156
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/29">Semantics</category>
|
|
157
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/19">Theory</category>
|
|
158
|
+
<pubDate>Wed, 22 Oct 2014 09:57:47 +0000</pubDate>
|
|
159
|
+
</item>
|
|
160
|
+
<item>
|
|
161
|
+
<title>EATCS Award 2014: Gordon Plotkin</title>
|
|
162
|
+
<link>http://lambda-the-ultimate.org/node/5068</link>
|
|
163
|
+
<description><blockquote ><p >
|
|
164
|
+
Gordon Plotkin is renowned for his groundbreaking contributions to programming language semantics, which have helped to shape the landscape of theoretical computer science, and which have im-pacted upon the design of programming languages and their verification technologies. The in-fluence of his pioneering work on logical frameworks pervades modern proof technologies. In addition, he has made outstanding contributions in machine learning, automated theorem prov-ing, and computer-assisted reasoning. He is still active in research at the topmost level, with his current activities placing him at the forefront of fields as diverse as programming semantics, applied logic, and systems biology.</blockquote>
|
|
165
|
+
<p >
|
|
166
|
+
Well <a href="http://www.eatcs.org/index.php/component/content/article/1-news/1856-the-eatcs-award-2014-laudatio-for-martin-dyer">deserved</a>, of course. Congrats!</p></description>
|
|
167
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/6">General</category>
|
|
168
|
+
<pubDate>Sun, 12 Oct 2014 22:54:33 +0000</pubDate>
|
|
169
|
+
</item>
|
|
170
|
+
<item>
|
|
171
|
+
<title>CFP: Off-the-Beaten-Track (OBT) workshop at POPL 2015</title>
|
|
172
|
+
<link>http://lambda-the-ultimate.org/node/5064</link>
|
|
173
|
+
<description><p >Announcing the 2015 edition of the OBT workshop, to be co-located with POPL 2015, in Mumbai, India. Two-page paper submissions are due November 7, 2014.</p>
|
|
174
|
+
<p >From the web page (<a href="http://www.cs.rice.edu/~sc40/obt15/">http://www.cs.rice.edu/~sc40/obt15/</a>):</p>
|
|
175
|
+
<blockquote ><p >
|
|
176
|
+
Programming language researchers have the principles, tools, algorithms and abstractions to solve all kinds of problems, in all areas of computer science. However, identifying and evaluating new problems, particularly those that lie outside the typical core PL problems we all know and love, can be a significant challenge. This workshop's goal is to identify and discuss problems that do not often show up in our top conferences, but where programming language research can make a substantial impact. We hope fora like this will increase the diversity of problems that are studied by PL researchers and thus increase our community's impact on the world.</p>
|
|
177
|
+
<p >While many workshops associated with POPL have become more like mini-conferences themselves, this is an anti-goal for OBT. The workshop will be informal and structured to encourage discussion. We are at least as interested in problems as in solutions.
|
|
178
|
+
</p></blockquote></description>
|
|
179
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/1">LtU Forum</category>
|
|
180
|
+
<pubDate>Sun, 05 Oct 2014 15:59:57 +0000</pubDate>
|
|
181
|
+
</item>
|
|
182
|
+
<item>
|
|
183
|
+
<title>Domain settings</title>
|
|
184
|
+
<link>http://lambda-the-ultimate.org/node/5060</link>
|
|
185
|
+
<description><p >I am about to make some changes to the name server definitions. Since changes take time to propagate, you may have trouble reaching the site for awhile. If this happens, try using the .com domain instead of the preferred .org domain.</p></description>
|
|
186
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/3">Admin</category>
|
|
187
|
+
<pubDate>Sat, 04 Oct 2014 11:24:32 +0000</pubDate>
|
|
188
|
+
</item>
|
|
189
|
+
<item>
|
|
190
|
+
<title>sml-family.org</title>
|
|
191
|
+
<link>http://lambda-the-ultimate.org/node/5058</link>
|
|
192
|
+
<description><p >In his <A href="http://existentialtype.wordpress.com/2014/09/26/sml-family-org-up-and-running/">blog</A>, <A href="http://www.cs.cmu.edu/~rwh/">Bob Harper</A>, in joint effort with <A href="http://people.cs.uchicago.edu/~dbm/">Dave MacQueen</A> and <A href="http://www.lars.com/">Lars Bergstrom</A>, announces the launch of <A href="http://sml-family.org">sml-family.org</A>:</p>
|
|
193
|
+
<blockquote ><p >
|
|
194
|
+
The Standard ML Family project provides a home for online versions of various formal definitions of Standard ML, including the "Definition of Standard ML, Revised" (Standard ML 97). The site also supports coordination between different implementations of the Standard ML (SML) programming language by maintaining common resources such as the documentation for the <A href="http://sml-family.org/Basis/index.html">Standard ML Basis Library</A> and standard test suites. The goal is to increase compatibility and resource sharing between Standard ML implementations. </p>
|
|
195
|
+
<p >The site includes a <A href="http://sml-family.org/#History">history section</A> devoted to the history of ML, and of Standard ML in particular. This section will contain a collection of original source documents relating to the design of the language.
|
|
196
|
+
</p></blockquote></description>
|
|
197
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/5">Fun</category>
|
|
198
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/11">Functional</category>
|
|
199
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/7">History</category>
|
|
200
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/8">Implementation</category>
|
|
201
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/10">Paradigms</category>
|
|
202
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/29">Semantics</category>
|
|
203
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/19">Theory</category>
|
|
204
|
+
<pubDate>Tue, 30 Sep 2014 19:27:43 +0000</pubDate>
|
|
205
|
+
</item>
|
|
206
|
+
<item>
|
|
207
|
+
<title>Inferring algebraic effects</title>
|
|
208
|
+
<link>http://lambda-the-ultimate.org/node/5055</link>
|
|
209
|
+
<description><p ><a href="http://www.lmcs-online.org/ojs/viewarticle.php?id=1469&amp;layout=abstract">Logical methods in computer science</A> just published <a href="http://matija.pretnar.info/">Matija Pretnar</A>'s <A href="http://arxiv.org/pdf/1312.2334.pdf">latest take on algebraic effects and handlers</A>:</p>
|
|
210
|
+
<blockquote ><p >
|
|
211
|
+
We present a complete polymorphic effect inference algorithm for an ML-style language with handlers of not only exceptions, but of any other algebraic effect such as input &amp; output, mutable references and many others. Our main aim is to offer the programmer a useful insight into the effectful behaviour of programs. Handlers help here by cutting down possible effects and the resulting lengthy output that often plagues precise effect systems. Additionally, we present a set of methods that further simplify the displayed types, some even by deliberately hiding inferred information from the programmer.
|
|
212
|
+
</p></blockquote>
|
|
213
|
+
<p >Pretnar and <A href="http://andrej.com/">Bauer</A>'s <A href="http://eff-lang.org/">Eff</A> has made <A href="http://lambda-the-ultimate.org/node/4090">previous</A> <A href="http://lambda-the-ultimate.org/node/4481">appearances</A> here on LtU. Apart from the new fangled polymorphic effect system, this paper also contains an Eff tutorial. </p></description>
|
|
214
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/11">Functional</category>
|
|
215
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/8">Implementation</category>
|
|
216
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/10">Paradigms</category>
|
|
217
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/29">Semantics</category>
|
|
218
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/19">Theory</category>
|
|
219
|
+
<pubDate>Sat, 27 Sep 2014 23:16:37 +0000</pubDate>
|
|
220
|
+
</item>
|
|
221
|
+
<item>
|
|
222
|
+
<title>LtU's new server</title>
|
|
223
|
+
<link>http://lambda-the-ultimate.org/node/5053</link>
|
|
224
|
+
<description>Lambda the Ultimate is now running on a new, faster, more reliable server. The old one is now, uh... pining for the fjords.
|
|
225
|
+
|
|
226
|
+
This should resolve the increasingly frequent outages we've seen recently.
|
|
227
|
+
|
|
228
|
+
Because the old server had started failing, we didn't have time to do as much quality control on the migration as we would have liked. If anyone notices any issues with the site, please comment in this thread.
|
|
229
|
+
|
|
230
|
+
Currently known issues:
|
|
231
|
+
|
|
232
|
+
<ul>
|
|
233
|
+
<li><span style="text-decoration:line-through">Non-Latin UTF-8 characters apparently didn't survive the database migration correctly. This is a particular issue if you have a username containing non-Latin characters - you may not be able to log in currently.</span>
|
|
234
|
+
<li><span style="text-decoration:line-through">It's possible that some comments posted later on Monday don't appear on the new site</span>. (Resolved: there were no missing comments)
|
|
235
|
+
<li>New user signup emails are not yet working.
|
|
236
|
+
<li><span style="text-decoration:line-through">Due to DNS propagation, not everyone will see the new site immediately.</span> (Now resolved)
|
|
237
|
+
</ul>
|
|
238
|
+
|
|
239
|
+
The struck out issues have been resolved. The remaining issue, with user signup emails, should be resolved in the next few days.
|
|
240
|
+
</description>
|
|
241
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/3">Admin</category>
|
|
242
|
+
<pubDate>Tue, 23 Sep 2014 07:26:16 +0000</pubDate>
|
|
243
|
+
</item>
|
|
244
|
+
<item>
|
|
245
|
+
<title>Breaking the Complexity Barrier of Pure Functional Programs with Impure Data Structures</title>
|
|
246
|
+
<link>http://lambda-the-ultimate.org/node/5052</link>
|
|
247
|
+
<description><p ><a href="https://lirias.kuleuven.be/bitstream/123456789/201251/1/preliminary.pdf">Breaking the Complexity Barrier of Pure Functional Programs with Impure Data Structures</a> by Pieter Wuille and Tom Schrijvers:</p>
|
|
248
|
+
<blockquote ><p >Pure functional programming language offer many advantages over impure languages. Unfortunately, the absence of destructive update, imposes a complexity barrier. In imperative languages, there are algorithms and data structures with better complexity. We present our project for combining existing program transformation techniques to transform inefficient pure data structures into impure ones with better complexity. As a consequence, the programmer is not exposed to the impurity and retains the advantages of purity.</p></blockquote>
|
|
249
|
+
<p >This paper is along the same lines <a href="http://lambda-the-ultimate.org/node/4611">a question I asked a couple of years ago</a>. The idea here is to allow programming using immutable interfaces, and then automatically transform it into a more efficient mutable equivalent.</p></description>
|
|
250
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/11">Functional</category>
|
|
251
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/19">Theory</category>
|
|
252
|
+
<pubDate>Mon, 22 Sep 2014 14:10:48 +0000</pubDate>
|
|
253
|
+
</item>
|
|
254
|
+
<item>
|
|
255
|
+
<title>Inside the Wolfram Language</title>
|
|
256
|
+
<link>http://lambda-the-ultimate.org/node/5050</link>
|
|
257
|
+
<description><p >Video of Stephen Wolfram <a href="https://www.youtube.com/watch?v=EjCWdsrVcBM">showing off the Wolfram Language</a> and sharing his perspective on the design of the language at Strange Loop conference.</p></description>
|
|
258
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/6">General</category>
|
|
259
|
+
<pubDate>Sun, 21 Sep 2014 02:20:09 +0000</pubDate>
|
|
260
|
+
</item>
|
|
261
|
+
<item>
|
|
262
|
+
<title>What's in store for the most widely used language by discerning hackers?</title>
|
|
263
|
+
<link>http://lambda-the-ultimate.org/node/5048</link>
|
|
264
|
+
<description><p >Or, in other words, what's the <a href="https://lists.gnu.org/archive/html/emacs-devel/2014-09/msg00434.html">future of Emacs Lisp</a> (and unavoidable <a href="https://news.ycombinator.com/item?id=8328206">HN discussion)</a>. </p>
|
|
265
|
+
<p >The original message contains some interesting tidbits. I am not sure how the discussion on emacs-devel will develop. But speculating about things such as Guile elisp is, of course, our bailiwick.</p></description>
|
|
266
|
+
<category domain="http://lambda-the-ultimate.org/taxonomy/term/6">General</category>
|
|
267
|
+
<pubDate>Wed, 17 Sep 2014 05:14:55 +0000</pubDate>
|
|
268
|
+
</item>
|
|
269
|
+
</channel>
|
|
270
|
+
</rss>
|
|
271
|
+
|
|
272
|
+
---
|
|
273
|
+
|
|
274
|
+
feed.format: rss 2.0
|
|
275
|
+
feed.title: Lambda the Ultimate - Programming Languages Weblog
|
|
276
|
+
feed.url: http://lambda-the-ultimate.org
|
|
277
|
+
feed.summary: Programming languages news, articles and discussion
|
|
278
|
+
|
|
279
|
+
feed.items[0].title: John C Reynolds Doctoral Dissertation Award nominations for 2014
|
|
280
|
+
feed.items[0].url: http://lambda-the-ultimate.org/node/5088
|
|
281
|
+
|
|
282
|
+
## note: no guid; will fallback/reuse url
|
|
283
|
+
feed.items[0].guid: http://lambda-the-ultimate.org/node/5088
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
feed.items[1].title: Zélus : A Synchronous Language with ODEs
|
|
287
|
+
feed.items[1].url: http://lambda-the-ultimate.org/node/5087
|
|
288
|
+
|